diff --git a/Cargo.lock b/Cargo.lock index cbf8f31e2..ae18376e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -999,9 +999,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.66" +version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" +checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -1031,9 +1031,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.103" +version = "0.9.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 339090056..9cb04ccb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ libc = "0.2" log = "0.4" miniz_oxide = "0.6" mio = { version = "1", features = ["os-poll", "os-ext", "net"] } -openssl = "=0.10.66" +openssl = "=0.10.70" paste = "1.0" rustls = "0.21" rustls-native-certs = "0.6" diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 000000000..c4601322d --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,216 @@ +## Building from source +```sh +git clone -b cache-dev https://github.com/radiumb/pushpin.git +``` + +### Dependencies +--- +If you’re using a recent version of Debian or Ubuntu, then all dependencies can be installed via package management: +```sh +sudo apt-get install make pkg-config rustc cargo g++ libzmq3-dev libssl-dev libboost-dev qtbase5-dev +``` +If you’re on macOS, then all dependencies can be installed via Homebrew: +```sh +brew install pkg-config rust zeromq boost qt +``` +### Compiling +--- +Simply run make: +```sh +make +``` +Optionally, you can install: +```sh +sudo make install +``` + +## Running +Running on Debian and Ubuntu +```sh +sudo pushpin --loglevel=2,proxy:3 --logfile=/var/log/pushpin/pushpin.log +``` +loglevel=2 : only show warnings, infos. 3 : show debug messages + +## Configuration +Pushpin has two primary configuration files: `pushpin.conf` and routes. The `pushpin.conf` file refers to the routes file via the routesfile field: +```sh +[proxy] +routesfile=routes +``` +If a relative path is used, then the file is looked up relative to the location of `pushpin.conf`. +```sh +sudo cp examples/config/pushpin.conf /usr/local/etc/pushpin/pushpin.conf +``` +```sh +sudo cp examples/config/routes /usr/local/etc/pushpin/routes +``` +### pushpin.conf file +For general options, please refer to [this](https://pushpin.org/docs/configuration/#pushpinconf-file). +Options for cache are added into this file. +* cache enable flag (false=disable, true=enable) + ``` + cache_enable=true + ``` +#### backends +* http url path list of backends when failed to get response + ``` + http_backend_urls=http://localhost:7999/ws1,http://localhost:7999/ws2,http://localhost:7999/ws3 ... + ``` +* websocket url paths for cache clients to connect to pushpin + ``` + ws_backend_urls=ws://localhost:7999/ws1,ws://localhost:7999/ws2,ws://localhost:7999/ws3 ... + ``` +#### cache methods +* cache methods + ``` + ws_cache_methods=author_hasKey,author_hasSessionKeys,... + ws_subscribe_methods=beefy_subscribeJustifications,chain_subscribeAllHeads,... + (* means ALL methods, ex : ws_cache_methods=*) + ``` +#### cache refresh +* never timeout/delete methods with params
+ ref: List of methods that are not even delete/auto-refresh if it has the valid parameters + ``` + ws_never_timeout_methods=chain_getBlockHash,chain_getBlock,chain_getHeader,state_queryStorageAt,... + ``` +* cache auto-refresh shorter timeout methods
+ ref: List of methods with shorter auto-refresh timeout periods (5 seconds) + ``` + ws_auto_refresh_longer_timeout_methods=chain_getBlockHash,chain_getHeader,state_getKeysPaged,state_queryStorageAt,... + ``` +* cache auto-refresh longer timeout methods
+ ref: List of methods with longer auto-refresh timeout periods (60 seconds) + ``` + ws_refresh_longer_methods=state_getMetadata,... + ``` +* cache auto-refresh no delete methods
+ ref: List of methods that exist permanently in the cache list and do auto-refresh periodically + ``` + ws_refresh_unerase_methods=chain_getBlockHash,system_health,eth_getBalance,... + ``` +* cache no auto-refresh methods
+ ref: List of methods that need to be cached but not auto refreshed + ``` + ws_refresh_exclude_methods=state_getMetadata,... + ``` +* cache pass-through methods
+ ref: List of methods that are not even cached but pass though to backend i.e. don't reject + ``` + ws_refresh_passthrough_methods=state_getStorage,... + ``` +* null response methods
+ ref: List of methods that treat null responses as valid + ``` + ws_null_response_methods=chainHead_v1_unpin,... + ``` +#### cache keys +* important fields in request, used as a key to identify cache items + ``` + ws_cache_key = $request_json_value["method"]+$user_defined["request_args"]+$request_json_pair["jsonrpc"] + request_args = $request_json_value["params"] + (ex : $request_json_value["method"]="author_hasKey", $request_json_pair["jsonrpc"]="\"jsonrpc\":\"2.0\"") + ``` +* field name is used as ID in Request or Response (to support multi-protocol in future) + ``` + message_id_attribute="id" + ``` +* field name is used as Method in Request or Response (to support multi-protocol in future) + ``` + message_method_attribute="method" + ``` +* field name is used as Params in Request (to support never_timeout_methods_with_params) + ``` + message_params_attribute="params" + ``` +* field names are used as Error in Response + ``` + message_error_attributes=error,fault,bug + ``` +#### timeout of auto-refresh +* cache timeout seconds (default 20)
+ ref: Number of seconds to count auto-refresh timeout + ``` + ws_auto_refresh_cache_timeout_seconds=20 + ``` +* cache shorter timeout seconds (default 10)
+ ref: Number of seconds to count auto-refresh shorter timeout + ``` + ws_auto_refresh_shorter_timeout_seconds=10 + ``` +* cache longer timeout seconds (default 60)
+ ref: Number of seconds to count auto-refresh longer timeout + ``` + ws_auto_refresh_longer_timeout_seconds=60 + ``` +* cache auto-refresh access timeout seconds (default 30)
+ ref: Number of seconds to count time out, be used to delete cache items that are not referenced by any client for this timeout period + ``` + ws_auto_refresh_access_timeout_seconds=30 + ``` +#### count of cache items +* Maximum cache item count (default 3000) + ``` + cache_item_max_count=3000 + ``` +#### timeout +* time seconds to switch another backend for null response (default 10) + ``` + backend_switch_interval_seconds=10 + ``` +#### error +* Fields in the response identifying errors + ``` + message_error_attributes=error,fault,bug + ``` +#### prometheus +* prometheus restore allow seconds (default 300) + ``` + prometheus_restore_allow_seconds=250 + ``` +#### redis +* redis enable flag (default false) + ``` + redis_enable=true + ``` +* redis server ip address (default 127.0.0.1) + ``` + redis_host_addr=127.0.0.1 + ``` +* redis server port number (default 6379) + ``` + redis_port=6379 + ``` +* redis server port number (default 100) + ``` + redis_pool_count=100 + ``` +* Redis key prefix to identify each Pushpin instance when sharing Redis between multiple instances. + ``` + redis_key_header="pushpin1:" + ``` +* IP address of the replication master. If this is set to a valid IP address, Redis will act as a replica and attempt to connect to the specified master. + ``` + replica_master_addr="" + ``` +* replication master port number (default 6379). Specifies the port Redis should use to connect to the master. + ``` + replica_master_port=6379 + ``` +#### Counts +* groups for promethus count + ``` + ws_count_groups=ws_key_group,ws_index_group,ws_block_group,ws_state_subscribe,ws_cache_candidates + ws_key_group=author_hasKey,author_insertKey,... + ws_block_group=chain_getBlock,chain_getBlockHash,... + ws_state_subscribe=state_subscribeRuntimeVersion,state_subscribeStorage,... + ws_cache_candidates=author_hasKey,author_hasSessionKeys,... + ``` + +## Appendix +### Installing redis-server + ``` + sudo apt update + sudo apt install redis-server -y + sudo systemctl enable redis-server + sudo systemctl start redis-server + ``` diff --git a/Makefile b/Makefile index e37997bea..3eed9ff9d 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,13 @@ -cargo_flags = +# controlled leading whitespace, per the GNU make manual +nullstring := +space := $(nullstring) # end of the line + ifdef RELEASE -cargo_flags += --offline --locked --release +cargo_flags = $(space)--offline --locked --release endif -cargo_toolchain = ifdef TOOLCHAIN -cargo_toolchain += +$(TOOLCHAIN) +cargo_toolchain = $(space)+$(TOOLCHAIN) endif all: postbuild diff --git a/examples/config/pushpin.conf b/examples/config/pushpin.conf index 69b215c3e..095db1873 100644 --- a/examples/config/pushpin.conf +++ b/examples/config/pushpin.conf @@ -37,7 +37,7 @@ logdir=log log_level=2 # client full request header must fit in this buffer -client_buffer_size=8192 +client_buffer_size=81920 # maximum number of client connections client_maxconn=50000 @@ -108,12 +108,17 @@ sockjs_url=http://cdn.jsdelivr.net/sockjs/0.3.4/sockjs.min.js # pushpin will output a log message when a new version is available. report # mode helps the pushpin project build credibility, so please enable it if you # enjoy this software :) -updates_check=report +updates_check=off # use this field to identify your organization in updates requests. if left # blank, updates requests will be anonymous organization_name= +# prometheus report +prometheus_port=9001 + +workers=1 + [handler] # ipc permissions (octal) @@ -171,3 +176,72 @@ stats_report_interval=10 # stats output format stats_format=tnetstring + + +[cache] +cache_enable=true + +# backends +http_backend_urls=http://localhost:7999/ws1,http://localhost:7999/ws2,http://localhost:7999/ws3,http://localhost:7999/ws4,http://localhost:7999/ws5 +ws_backend_urls=ws://localhost:7999/ws1,ws://localhost:7999/ws2,ws://localhost:7999/ws3,ws://localhost:7999/ws4,ws://localhost:7999/ws5 + +# cache methods +ws_cache_methods=* +#author_hasKey,author_hasSessionKeys,author_insertKey,author_pendingExtrinsics,author_removeExtrinsic,author_rotateKeys,author_submitExtrinsic,babe_epochAuthorship,beefy_getFinalizedHead,chain_getBlock,chain_getBlockHash,chain_getFinalizedHead,chain_getHeader,childstate_getKeys,childstate_getKeysPaged,childstate_getStorage,childstate_getStorageEntries,childstate_getStorageHash,childstate_getStorageSize,contracts_call,contracts_getStorage,contracts_instantiate,contracts_rentProjection,contracts_upload_code,dev_getBlockStats,engine_createBlock,engine_finalizeBlock,eth_accounts,eth_blockNumber,eth_call,eth_chainId,eth_coinbase,eth_estimateGas,eth_feeHistory,eth_gasPrice,eth_getBalance,eth_getBlockByHash,eth_getBlockByNumber,eth_getBlockTransactionCountByHash,eth_getBlockTransactionCountByNumber,eth_getCode,eth_getFilterChanges,eth_getFilterLogs,eth_getLogs,eth_getProof,eth_getStorageAt,eth_getTransactionByBlockHashAndIndex,eth_getTransactionByBlockNumberAndIndex,eth_getTransactionByHash,eth_getTransactionCount,eth_getTransactionReceipt,eth_getUncleByBlockHashAndIndex,eth_getUncleCountByBlockHash,eth_getUncleCountByBlockNumber,eth_getWork,eth_hashrate,eth_maxPriorityFeePerGas,eth_mining,eth_newBlockFilter,eth_newFilter,eth_newPendingTransactionFilter,eth_protocolVersion,eth_sendRawTransaction,eth_sendTransaction,eth_submitHashrate,eth_submitWork,eth_syncing,eth_uninstallFilter,net_listening,net_version,web3_clientVersion,web3_sha3,grandpa_proveFinality,grandpa_roundState,mmr_generateBatchProof,mmr_generateProof,offchain_localStorageGet,offchain_localStorageSet,payment_queryFeeDetails,payment_queryInfo,rpc_methods,state_call,state_getChildKeys,state_getChildReadProof,state_getChildStorage,state_getChildStorageHash,state_getChildStorageSize,state_getKeys,state_getKeysPaged,state_getMetadata,state_getPairs,state_getReadProof,state_getRuntimeVersion,state_getStorage,state_getStorageHash,state_getStorageSize,state_queryStorage,state_queryStorageAt,state_traceBlock,state_trieMigrationStatus,sync_state_genSyncSpec,system_accountNextIndex,system_addLogFilter,system_addReservedPeer,system_chain,system_chainType,system_dryRun,system_health,system_localListenAddresses,system_localPeerId,system_name,system_networkState,system_nodeRoles,system_peers,system_properties,system_removeReservedPeer,system_reservedPeers,system_resetLogFilter,system_syncState,system_version +ws_subscribe_methods=author_submitAndWatchExtrinsic+author_unwatchExtrinsic,beefy_subscribeJustifications+beefy_unsubscribeJustifications,chain_subscribeAllHeads+chain_unsubscribeAllHeads,chain_subscribeFinalisedHeads+chain_unsubscribeFinalisedHeads,chain_subscribeFinalizedHeads+chain_unsubscribeFinalizedHeads,chain_subscribeNewHead+chain_unsubscribeNewHead,chain_subscribeNewHeads+chain_unsubscribeNewHeads,chain_subscribeRuntimeVersion+chain_unsubscribeRuntimeVersion,grandpa_subscribeJustifications+grandpa_unsubscribeJustifications,state_subscribeRuntimeVersion+state_unsubscribeRuntimeVersion,state_subscribeStorage+state_unsubscribeStorage,transaction_unstable_submitAndWatch+transaction_unstable_unwatch,chainHead_v1_follow+chainHead_v1_unfollow + +# cache refresh +ws_never_timeout_methods=chain_getBlockHash,chain_getBlock,chain_getHeader,state_queryStorageAt +ws_refresh_shorter_methods=chain_getBlockHash,chain_getHeader,state_getKeysPaged,state_queryStorageAt +ws_refresh_longer_methods=state_getMetadata +ws_refresh_unerase_methods=chain_getBlockHash,system_health,eth_getBalance +ws_refresh_exclude_methods=state_getMetadata +ws_refresh_passthrough_methods=state_getStorage +ws_null_response_methods=chainHead_v1_unpin + +# cache key +ws_cache_key = $request_json_value["method"] + $user_defined["request_args"]+$request_json_pair["jsonrpc"] +request_args = $request_json_value["params"] +message_id_attribute="id" +message_method_attribute="method" +message_params_attribute="params" + +# auto-refresh. timeout +# cache timeout seconds (default 20) +ws_auto_refresh_cache_timeout_seconds=20 +# cache shorter timeout seconds (default 10) +ws_auto_refresh_shorter_cache_timeout_seconds=10 +# cache longer timeout seconds (default 60) +ws_auto_refresh_longer_timeout_seconds=60 +# cache auto-refresh access timeout seconds (default 30) +ws_auto_refresh_access_timeout_seconds=30 + +# Maximum cache item count (default 3000) +cache_item_max_count=3000 + +# time seconds to retry another backend for null response (default 10) +backend_switch_interval_seconds=10 + +# error +message_error_attributes=error,fault,bug + +# prometheus restore allow seconds (default 300) +prometheus_restore_allow_seconds=250 + +# redis +redis_enable=false +redis_host_addr=127.0.0.1 +redis_port=6379 +redis_pool_count=100 +redis_key_header="pushpin1:" +replica_master_addr="107.155.71.83" +replica_master_port=6379 + +# groups to count +ws_count_groups=ws_key_group,ws_index_group,ws_block_group,ws_chain_group,ws_state_subscribe,ws_cache_candidates +ws_key_group=author_hasKey,author_insertKey,childstate_getKeys,childstate_getKeysPaged,state_getChildKeys,state_getKeys, +ws_index_group=eth_getTransactionByBlockHashAndIndex,eth_getTransactionByBlockNumberAndIndex,eth_getUncleByBlockHashAndIndex,eth_getUncleByBlockNumberAndIndex,system_accountNextIndex +ws_block_group=chain_getBlock,chain_getBlockHash,dev_getBlockStats,engine_createBlock,engine_finalizeBlock,eth_blockNumber,eth_getBlockByHash,eth_getBlockByNumber,eth_getBlockTransactionCountByHash,eth_getBlockTransactionCountByNumber,eth_getTransactionByBlockHashAndIndex +ws_chain_group=chain_getBlock,chain_getBlockHash,chain_getFinalizedHead,chain_getHeader,chain_subscribeAllHeads,chain_subscribeFinalizedHeads,chain_subscribeNewHead +ws_state_subscribe=state_subscribeRuntimeVersion,state_subscribeStorage +ws_cache_candidates=author_hasKey,author_hasSessionKeys,author_insertKey,author_pendingExtrinsics,author_removeExtrinsic,author_rotateKeys,author_submitExtrinsic,babe_epochAuthorship,beefy_getFinalizedHead,chain_getBlock,chain_getBlockHash,chain_getFinalizedHead,chain_getHeader,childstate_getKeys,childstate_getKeysPaged,childstate_getStorage,childstate_getStorageEntries,childstate_getStorageHash,childstate_getStorageSize,contracts_call,contracts_getStorage,contracts_instantiate,contracts_rentProjection,contracts_upload_code,dev_getBlockStats,engine_createBlock,engine_finalizeBlock,eth_accounts,eth_blockNumber,eth_call,eth_chainId,eth_coinbase,eth_estimateGas,eth_feeHistory,eth_gasPrice,eth_getBalance,eth_getBlockByHash,eth_getBlockByNumber,eth_getBlockTransactionCountByHash,eth_getBlockTransactionCountByNumber,eth_getCode,eth_getFilterChanges,eth_getFilterLogs,eth_getLogs,eth_getProof,eth_getStorageAt,eth_getTransactionByBlockHashAndIndex,eth_getTransactionByBlockNumberAndIndex,eth_getTransactionByHash,eth_getTransactionCount,eth_getTransactionReceipt,eth_getUncleByBlockHashAndIndex,eth_getUncleCountByBlockHash,eth_getUncleCountByBlockNumber,eth_getWork,eth_hashrate,eth_maxPriorityFeePerGas,eth_mining,eth_newBlockFilter,eth_newFilter,eth_newPendingTransactionFilter,eth_protocolVersion,eth_sendRawTransaction,eth_sendTransaction,eth_submitHashrate,eth_submitWork,eth_syncing,eth_uninstallFilter,net_listening,net_version,web3_clientVersion,web3_sha3,grandpa_proveFinality,grandpa_roundState,mmr_generateBatchProof,mmr_generateProof,offchain_localStorageGet,offchain_localStorageSet,payment_queryFeeDetails,payment_queryInfo,rpc_methods,state_call,state_getChildKeys,state_getChildReadProof,state_getChildStorage,state_getChildStorageHash,state_getChildStorageSize,state_getKeys,state_getKeysPaged,state_getMetadata,state_getPairs,state_getReadProof,state_getRuntimeVersion,state_getStorage,state_getStorageHash,state_getStorageSize,state_queryStorage,state_queryStorageAt,state_traceBlock,state_trieMigrationStatus,sync_state_genSyncSpec,system_accountNextIndex,system_addLogFilter,system_addReservedPeer,system_chain,system_chainType,system_dryRun,system_health,system_localListenAddresses,system_localPeerId,system_name,system_networkState,system_nodeRoles,system_peers,system_properties,system_removeReservedPeer,system_reservedPeers,system_resetLogFilter,system_syncState,system_version,state_subscribeStorage,state_subscribeRuntimeVersion,chain_subscribeNewHead,chain_subscribeFinalizedHeads diff --git a/examples/config/routes b/examples/config/routes index 1e3550b35..cd3141e97 100644 --- a/examples/config/routes +++ b/examples/config/routes @@ -1 +1,2 @@ -* test +*,proto=ws,no_grip,debug localhost:8999 +*,proto=http,no_grip,debug localhost:8999 diff --git a/src/connmgr/batch.rs b/src/connmgr/batch.rs index ec123fc0f..bdf773daf 100644 --- a/src/connmgr/batch.rs +++ b/src/connmgr/batch.rs @@ -147,7 +147,7 @@ impl Batch { self.nodes.remove(key.nkey); } - pub fn take_group<'a, 'b: 'a, F>(&'a mut self, get_id: F) -> Option + pub fn take_group<'a, 'b: 'a, F>(&'a mut self, get_id: F) -> Option> where F: Fn(usize) -> Option<(&'b [u8], u32)>, { diff --git a/src/connmgr/client.rs b/src/connmgr/client.rs index 667c02ea4..a83fd08ee 100644 --- a/src/connmgr/client.rs +++ b/src/connmgr/client.rs @@ -1,6 +1,6 @@ /* * Copyright (C) 2023 Fanout, Inc. - * Copyright (C) 2023 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -634,7 +634,7 @@ impl Worker { let instance_id = Rc::new(instance_id); - let ka_batch = (stream_maxconn + (KEEP_ALIVE_BATCHES - 1)) / KEEP_ALIVE_BATCHES; + let ka_batch = stream_maxconn.div_ceil(KEEP_ALIVE_BATCHES); let batch = Batch::new(ka_batch); @@ -1891,9 +1891,12 @@ impl TestClient { let (status_s, status_r) = channel::channel(1000); let (control_s, control_r) = channel::channel(1000); - let thread = thread::spawn(move || { - Self::run(status_s, control_r, zmq_context); - }); + let thread = thread::Builder::new() + .name("test-client".to_string()) + .spawn(move || { + Self::run(status_s, control_r, zmq_context); + }) + .unwrap(); // wait for handler thread to start assert_eq!(status_r.recv().unwrap(), StatusMessage::Started); diff --git a/src/connmgr/connection.rs b/src/connmgr/connection.rs index 3209f7d5d..776b6c868 100644 --- a/src/connmgr/connection.rs +++ b/src/connmgr/connection.rs @@ -527,7 +527,7 @@ pub struct AddrRef<'a> { s: Ref<'a, Option>>, } -impl<'a> AddrRef<'a> { +impl AddrRef<'_> { pub fn get(&self) -> Option<&[u8]> { match &*self.s { Some(s) => Some(s.as_slice()), @@ -701,8 +701,8 @@ struct SendMessageContentFuture<'a, 'b, W: AsyncWrite, M> { done: bool, } -impl<'a, 'b, W: AsyncWrite, M: AsRef<[u8]> + AsMut<[u8]>> Future - for SendMessageContentFuture<'a, 'b, W, M> +impl + AsMut<[u8]>> Future + for SendMessageContentFuture<'_, '_, W, M> { type Output = Result<(usize, bool), Error>; @@ -1325,7 +1325,7 @@ where } } -async fn send_error_response<'a, R: AsyncRead, W: AsyncWrite>( +async fn send_error_response( mut resp: server::Response<'_, R, W>, zreceiver: &TrackedAsyncLocalReceiver<'_, (arena::Rc, usize)>, e: &Error, @@ -4210,7 +4210,7 @@ enum AsyncStream<'a> { Tls(AsyncTlsStream<'a>), } -impl<'a> AsyncStream<'a> { +impl AsyncStream<'_> { fn into_inner(self) -> Stream { match self { Self::Plain(stream) => Stream::Plain(stream.into_std()), diff --git a/src/connmgr/server.rs b/src/connmgr/server.rs index a86466ce7..bd53ecce5 100644 --- a/src/connmgr/server.rs +++ b/src/connmgr/server.rs @@ -1,6 +1,6 @@ /* * Copyright (C) 2020-2023 Fanout, Inc. - * Copyright (C) 2023 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -777,7 +777,7 @@ impl Worker { let instance_id = Rc::new(instance_id); - let ka_batch = (stream_maxconn + (KEEP_ALIVE_BATCHES - 1)) / KEEP_ALIVE_BATCHES; + let ka_batch = stream_maxconn.div_ceil(KEEP_ALIVE_BATCHES); let batch = Batch::new(ka_batch); @@ -2305,9 +2305,12 @@ impl TestServer { let (started_s, started_r) = channel::channel(1); let (stop_s, stop_r) = channel::channel(1); - let thread = thread::spawn(move || { - Self::run(started_s, stop_r, zmq_context); - }); + let thread = thread::Builder::new() + .name("test-server".to_string()) + .spawn(move || { + Self::run(started_s, stop_r, zmq_context); + }) + .unwrap(); // wait for handler thread to start started_r.recv().unwrap(); diff --git a/src/connmgr/tls.rs b/src/connmgr/tls.rs index 12be35949..42a33797f 100644 --- a/src/connmgr/tls.rs +++ b/src/connmgr/tls.rs @@ -201,10 +201,7 @@ impl IdentityCache { return Some(identity); } - let pos = match name.find('.') { - Some(pos) => pos, - None => return None, - }; + let pos = name.find('.')?; let name = format!("_{}", &name[pos..]); @@ -1085,7 +1082,7 @@ impl<'a: 'b, 'b> AsyncTlsStream<'a> { } } -impl<'a> Drop for AsyncTlsStream<'a> { +impl Drop for AsyncTlsStream<'_> { fn drop(&mut self) { let registration = self.waker.take_registration(); diff --git a/src/connmgr/track.rs b/src/connmgr/track.rs index e2e6db2a3..edebbf8da 100644 --- a/src/connmgr/track.rs +++ b/src/connmgr/track.rs @@ -70,7 +70,7 @@ impl<'a, A, B> Track<'a, (A, B)> { } } -impl<'a, T> Drop for Track<'a, T> { +impl Drop for Track<'_, T> { fn drop(&mut self) { if let Some(inner) = &self.inner { inner.active.set(false); @@ -78,7 +78,7 @@ impl<'a, T> Drop for Track<'a, T> { } } -impl<'a, T> Deref for Track<'a, T> { +impl Deref for Track<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -134,7 +134,7 @@ pub struct TrackFuture<'a, F> { value_active: &'a TrackFlag, } -impl<'a, F, T, E> Future for TrackFuture<'a, F> +impl Future for TrackFuture<'_, F> where F: Future>, E: From, diff --git a/src/core/cacheutil.cpp b/src/core/cacheutil.cpp new file mode 100644 index 000000000..e4d213fd1 --- /dev/null +++ b/src/core/cacheutil.cpp @@ -0,0 +1,2157 @@ +/* + * Copyright (C) 2017-2022 Fanout, Inc. + * Copyright (C) 2024 Fastly, Inc. + * + * This file is part of Pushpin. + * + * $FANOUT_BEGIN_LICENSE:APACHE2$ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $FANOUT_END_LICENSE$ + */ + +#include "cacheutil.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qtcompat.h" +#include "tnetstring.h" +#include "log.h" + +extern bool gCacheEnable; +extern QStringList gHttpBackendUrlList; +extern QStringList gWsBackendUrlList; + +// Response Body for cache item +QMap gCacheResponseBuffer; + +QHash gCacheItemMap; + +extern QString gMsgIdAttrName; +extern QString gMsgMethodAttrName; +extern QString gMsgParamsAttrName; +extern QString gResultAttrName; +extern QStringList gErrorAttrList; +extern QString gSubscriptionAttrName; +extern QString gSubscribeBlockAttrName; +extern QString gSubscribeChangesAttrName; + +extern QStringList gCacheMethodList; +extern QHash gSubscribeMethodMap; +extern QHash> gUnsubscribeRequestMap; +extern QList gDeleteClientList; +extern QStringList gNeverTimeoutMethodList; +extern QList gCacheKeyItemList; + +// multi packets params +extern QHash gHttpMultiPartResponseItemMap; +extern QHash gWsMultiPartRequestItemMap; +extern QHash gWsMultiPartResponseItemMap; + +extern QList gWsCacheClientList; +extern QHash gWsClientMap; +extern QHash gHttpClientMap; + +extern int gAccessTimeoutSeconds; +extern int gResponseTimeoutSeconds; +extern int gClientNoRequestTimeoutSeconds; +extern int gCacheTimeoutSeconds; +extern int gShorterTimeoutSeconds; +extern int gLongerTimeoutSeconds; +extern int gCacheItemMaxCount; + +extern int gBackendSwitchIntervalSeconds; + +// path of prometheus backup file +static QString gPrometheusBackupDir = "/var/log/"; +extern int gPrometheusRestoreAllowSeconds; + +// redis +extern bool gRedisEnable; +extern QString gRedisKeyHeader; +extern bool gReplicaFlag; +extern QString gReplicaMasterAddr; +extern int gReplicaMasterPort; + +// count method group +extern QHash gCountMethodGroupMap; + +// definitions for cache +#define MAGIC_STRING "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + +// prometheus status +extern quint32 numRequestReceived, numMessageSent, numWsConnect; +extern quint32 numClientCount, numHttpClientCount, numWsClientCount; +extern quint32 numRpcAuthor, numRpcBabe, numRpcBeefy, numRpcChain, numRpcChildState; +extern quint32 numRpcContracts, numRpcDev, numRpcEngine, numRpcEth, numRpcNet; +extern quint32 numRpcWeb3, numRpcGrandpa, numRpcMmr, numRpcOffchain, numRpcPayment; +extern quint32 numRpcRpc, numRpcState, numRpcSyncstate, numRpcSystem, numRpcSubscribe; +extern quint32 numCacheInsert, numCacheHit, numNeverTimeoutCacheInsert, numNeverTimeoutCacheHit; +extern quint32 numCacheLookup, numCacheExpiry, numRequestMultiPart; +extern quint32 numSubscriptionInsert, numSubscriptionHit, numSubscriptionLookup, numSubscriptionExpiry, numResponseMultiPart; +extern quint32 numCacheItem, numAutoRefreshItem, numAREItemCount, numSubscriptionItem, numNeverTimeoutCacheItem; +extern QHash groupMethodCountMap; + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// HiRedis +void redis_removeall_cache_item() +{ + log_debug("[REDIS] redis_removeall_cache_item"); + auto conn = RedisPool::instance()->acquire(); + + if (!conn) + { + log_debug("[REDIS] CONN failed\n"); + return; + } + + redisReply* reply = (redisReply*)redisCommand(conn.data(), "FLUSHDB"); + + if (reply->type == REDIS_REPLY_STATUS && std::string(reply->str) == "OK") + { + log_debug("[REDIS] Database cleared successfully."); + } + + if (reply != nullptr) + freeReplyObject(reply); + //pool.release(conn); + + return; +} + +void redis_reset_replica() +{ + log_debug("[REDIS] redis_reset_replica"); + auto conn = RedisPool::instance()->acquire(); + + if (!conn) + { + log_debug("[REDIS] CONN failed\n"); + return; + } + + redisReply* reply = (redisReply*)redisCommand(conn.data(), "REPLICAOF NO ONE"); + + if (reply->type == REDIS_REPLY_STATUS && std::string(reply->str) == "OK") + { + log_debug("[REDIS] REPLICAOF NO ONE."); + } + + if (reply != nullptr) + freeReplyObject(reply); + + char cmdStr[256]; + sprintf(cmdStr, "REPLICAOF %s %d", qPrintable(gReplicaMasterAddr), gReplicaMasterPort); + reply = (redisReply*)redisCommand(conn.data(), cmdStr); + + if (reply->type == REDIS_REPLY_STATUS && std::string(reply->str) == "OK") + { + log_debug("[REDIS] %s.", cmdStr); + } + + if (reply != nullptr) + freeReplyObject(reply); + //pool.release(conn); + + return; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Cache Item +bool is_cache_item(const QByteArray& itemId, QString methodName) +{ + // global cache item map + bool ret = gCacheItemMap.contains(itemId); + + if (ret == false && gRedisEnable == true) + { + // try to read from redis + QByteArray buff = redis_load_cache_response(itemId); + if (buff.length() > 0) + { + // create new cache item + struct CacheItem cacheItem; + + cacheItem.refreshFlag = 0x00; + cacheItem.cachedFlag = true; + cacheItem.proto = Scheme::none; + cacheItem.retryCount = 0; + cacheItem.cacheClientId = QByteArray(""); + cacheItem.methodName = methodName; + cacheItem.subscriptionUpdateCount = 0; + + // check cache/subscribe method + if (is_cache_method(methodName)) + { + cacheItem.methodType = CACHE_METHOD; + } + else if (is_subscribe_method(methodName)) + { + cacheItem.methodType = SUBSCRIBE_METHOD; + //return false; + } + + create_cache_item(itemId, cacheItem); + ret = gCacheItemMap.contains(itemId); + } + } + + return ret; +} + +CacheItem* load_cache_item(const QByteArray& itemId, QString methodName) +{ + CacheItem* ret = NULL; + + // global cache item map + if (!is_cache_item(itemId, methodName)) + { + log_debug("[CACHE] not found cache item %s", itemId.toHex().data()); + return NULL; + } + ret = &gCacheItemMap[itemId]; + + return ret; +} + +void create_cache_item(const QByteArray& itemId, const CacheItem& cacheItem) +{ + // global cache item map + gCacheItemMap[itemId] = cacheItem; + + // save prometheus + if (cacheItem.methodType == CACHE_METHOD) + { + if ((cacheItem.refreshFlag & AUTO_REFRESH_NEVER_TIMEOUT) != 0) + numNeverTimeoutCacheInsert++; + else + numCacheInsert++; + } + else if (cacheItem.methodType == SUBSCRIBE_METHOD) + { + numSubscriptionInsert++; + } + + return; +} + +void remove_cache_item(const QByteArray& itemId) +{ + // prometheus status + if (gCacheItemMap[itemId].methodType == CACHE_METHOD) + { + numCacheExpiry++; + } + else if (gCacheItemMap[itemId].methodType == SUBSCRIBE_METHOD) + { + if (gCacheItemMap[itemId].newMsgId != -1) + numSubscriptionExpiry++; + } + + if (gRedisEnable == true) + { + log_debug("[REDIS] remove cache item %s", itemId.toHex().data()); + redis_remove_item(itemId); + + if (gCacheItemMap[itemId].methodType == SUBSCRIBE_METHOD) + { + // remove subscription + QByteArray subscriptionKey = itemId + "-sub"; + redis_remove_item(subscriptionKey); + + // remove update + QByteArray updateKey = itemId + "-update"; + redis_remove_item(updateKey); + + // remove updateCount + QByteArray updateCountKey = itemId + "-updateCount"; + redis_remove_item(updateCountKey); + } + } + + if (is_cache_item(itemId)) + { + log_debug("[CACHE] remove cache item %s", itemId.toHex().data()); + gCacheItemMap.remove(itemId); + } + + return; +} + +QList get_cache_item_ids() +{ + return gCacheItemMap.keys(); +} + +void redis_remove_item(const QByteArray& itemId) +{ + auto conn = RedisPool::instance()->acquire(); + + if (!conn) + { + log_debug("[REDIS] CONN failed\n"); + return; + } + + QString key = gRedisKeyHeader + itemId.toHex().constData(); + + redisReply* reply = (redisReply*)redisCommand(conn.data(), + "DEL %s", + qPrintable(key) + ); + + if (reply != nullptr) + freeReplyObject(reply); +} + +void redis_store_cache_response(const QByteArray& itemId, const QByteArray& response) +{ + auto conn = RedisPool::instance()->acquire(); + + if (!conn) + { + log_debug("[REDIS] CONN failed\n"); + return; + } + + QString key = gRedisKeyHeader + itemId.toHex().constData(); + + redisReply* reply = (redisReply*)redisCommand(conn.data(), + "SET %s %b", + qPrintable(key), + response.constData(), response.size() + ); + + if (reply != nullptr) + freeReplyObject(reply); +} + +QByteArray redis_load_cache_response(const QByteArray& itemId) +{ + auto conn = RedisPool::instance()->acquire(); + + if (!conn) + { + log_debug("[REDIS] CONN failed\n"); + return QByteArray(); + } + + QString key = gRedisKeyHeader + itemId.toHex().constData(); + + redisReply* reply = (redisReply*)redisCommand(conn.data(), + "GET %s", + qPrintable(key) + ); + + if (reply == nullptr) + return QByteArray(); + + QByteArray response(reply->str, reply->len); + + if (reply != nullptr) + freeReplyObject(reply); + + return response; +} + +void store_cache_response_buffer(const QByteArray& itemId, const QByteArray& responseBuf, QString msgId, int addLen) +{ + QByteArray buff = responseBuf; + + // remove connmgr Txxx: + QByteArray prefix = " T"; + int start = buff.indexOf(prefix); + if (start != -1) + { + int colon = buff.indexOf(':', start + prefix.length()); + if (colon != -1) + { + buff.remove(0, colon + 1); // Remove up to and including colon + } + } + + // replace id + prefix = "2:id,"; + start = buff.indexOf(prefix); + if (start != -1) + { + int end = buff.indexOf(',', start + prefix.length()); + if (end != -1) + { + buff.replace(start, end - start, "2:id,__ID__"); // Replace + } + } + + // replace seq + prefix = "3:seq,"; + start = buff.indexOf(prefix); + if (start != -1) + { + int end = buff.indexOf('#', start + prefix.length()); + if (end != -1) + { + buff.replace(start, end - start, "3:seq,__SEQ__"); // Replace + } + } + + // replace from + prefix = "4:from,"; + start = buff.indexOf(prefix); + if (start != -1) + { + int end = buff.indexOf(',', start + prefix.length()); + if (end != -1) + { + buff.replace(start, end - start, "4:from,__FROM__"); // Replace + } + } + + // replace body length + int msgIdLen = !msgId.isEmpty() ? msgId.length() : 0; + int bodyLen = 0; + prefix = "4:body,"; + start = buff.indexOf(prefix); + if (start != -1) + { + int end = buff.indexOf(':', start + prefix.length()); + QByteArray part = buff.mid(start + prefix.length(), end-start-prefix.length()); + bodyLen = part.toInt(); + QByteArray newPattern = QByteArray("4:body,__BODY__") + QByteArray::number(bodyLen-msgIdLen+addLen); + if (end != -1) + { + buff.replace(start, end - start, newPattern); // Replace + } + } + + if (!msgId.isEmpty()) + { + // replace msgId + QByteArray oldPattern = QByteArray("\"id\":") + msgId.toUtf8(); + QByteArray newPattern = QByteArray("\"id\":__MSGID__"); + buff.replace(oldPattern, newPattern); + + // replace Content-Length header + int bodyLenNumLength = QString::number(bodyLen).length(); + oldPattern = QByteArray("14:Content-Length,") + QByteArray::number(bodyLenNumLength) + QByteArray(":") + QByteArray::number(bodyLen); + newPattern = QByteArray("14:Content-Length,__CONTENT_LENGTH__"); + buff.replace(oldPattern, newPattern); + } + + log_debug("[STORE_BUFF]-%s %s", itemId.constData(), buff.constData()); + + if (gRedisEnable == false) + { + gCacheResponseBuffer[itemId] = buff; + } + else + { + gCacheResponseBuffer[itemId] = buff; + redis_store_cache_response(itemId, buff); + QThread::usleep(1); + } +} + +QByteArray load_cache_response_buffer(const QByteArray& instanceAddress, const QByteArray& itemId, QByteArray packetId, int seqNum, QString msgId, QByteArray from, int addLen=0) +{ + QByteArray buff = ""; + if (gRedisEnable == false) + { + buff = gCacheResponseBuffer[itemId]; + } + else + { + buff = redis_load_cache_response(itemId); + if (buff.length() == 0) + { + log_debug("[REDIS] missed response buffer"); + buff = gCacheResponseBuffer[itemId]; + redis_store_cache_response(itemId, buff); + } + } + + // replace id + int idLen = packetId.length(); + QByteArray oldPattern = QByteArray("2:id,") + QByteArray("__ID__"); + QByteArray newPattern = QByteArray("2:id,") + QByteArray::number(idLen) + QByteArray(":") + packetId; + buff.replace(oldPattern, newPattern); + + // replace seq + int seqNumLength = QString::number(seqNum).length(); + oldPattern = QByteArray("3:seq,") + QByteArray("__SEQ__"); + newPattern = QByteArray("3:seq,") + QByteArray::number(seqNumLength) + QByteArray(":") + QByteArray::number(seqNum); + buff.replace(oldPattern, newPattern); + + // replace from + int fromLen = from.length(); + oldPattern = QByteArray("4:from,") + QByteArray("__FROM__"); + newPattern = QByteArray("4:from,") + QByteArray::number(fromLen) + QByteArray(":") + from; + buff.replace(oldPattern, newPattern); + + // replace msgId/bodyLen + int startIndex = buff.indexOf("\"id\":__MSGID__"); + if (startIndex >= 0) + { + // replace msgId + oldPattern = QByteArray("\"id\":__MSGID__"); + newPattern = QByteArray("\"id\":") + msgId.toUtf8(); + buff.replace(oldPattern, newPattern); + + // replace bodyLen + int msgIdLen = msgId.length(); + int newLen = 0; + startIndex = buff.indexOf("4:body,__BODY__"); + if (startIndex >= 0) + { + startIndex += 15; + int endIndex = buff.indexOf(':', startIndex); + QByteArray part = buff.mid(startIndex, endIndex-startIndex); + int orgLen = part.toInt(); + newLen = orgLen + msgIdLen + addLen; + newPattern = QByteArray("4:body,") + QByteArray::number(newLen); + buff.replace(startIndex-15, endIndex-startIndex+15, newPattern); + } + + // replace Content-Length header + oldPattern = QByteArray("14:Content-Length,__CONTENT_LENGTH__"); + int bodyLenNumLength = QString::number(newLen).length(); + newPattern = QByteArray("14:Content-Length,") + QByteArray::number(bodyLenNumLength) + QByteArray(":") + QByteArray::number(newLen); + buff.replace(oldPattern, newPattern); + } + else + { + // replace bodyLen + int newLen = 0; + startIndex = buff.indexOf("4:body,__BODY__"); + if (startIndex >= 0) + { + startIndex += 15; + int endIndex = buff.indexOf(':', startIndex); + QByteArray part = buff.mid(startIndex, endIndex-startIndex); + int orgLen = part.toInt(); + newLen = orgLen + addLen; + newPattern = QByteArray("4:body,") + QByteArray::number(newLen); + buff.replace(startIndex-15, endIndex-startIndex+15, newPattern); + } + } + + // add connmgr Txxx: + int buffLen = buff.length(); + buff = instanceAddress + " T" + QByteArray::number(buffLen-1) + QByteArray(":") + buff; + + log_debug("[LOAD_BUFF]-%s %s", itemId.constData(), buff.constData()); + + return buff; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Cache Thread +bool gCacheThreadAllowFlag = true; +static int gMainThreadRunning = 0; +static bool gCacheThreadRunning = false; + +void pause_cache_thread() +{ + if (gMainThreadRunning) + { + gMainThreadRunning++; + return; + } + + while (gCacheThreadRunning) + { + QThread::usleep(1); + } + + gMainThreadRunning++; +} + +void resume_cache_thread() +{ + if (!gMainThreadRunning) + { + return; + } + + gMainThreadRunning--; +} + +static void remove_old_cache_items() +{ + qint64 currMTime = QDateTime::currentMSecsSinceEpoch(); + qint64 accessTimeoutMSeconds = gAccessTimeoutSeconds * 1000; + qint64 responseTimeoutMSeconds = gResponseTimeoutSeconds * 1000; + + bool checkFlag = true; + + while (accessTimeoutMSeconds > 0) + { + QList cacheItemIdList = gCacheItemMap.keys();//get_cache_item_ids(); + QList deleteIdList; + int itemCount = cacheItemIdList.count(); + + int cacheItemCount = 0; + int subscribeItemCount = 0; + int neverTimeoutCacheItemCount = 0; + int autoRefreshItemCount = 0; + + // Remove items where the value is greater than 30 + for (int i=0; i < itemCount; i++) + { + QByteArray itemId = cacheItemIdList[i]; + CacheItem *pCacheItem = &gCacheItemMap[itemId]; + + if (pCacheItem->methodType == CacheMethodType::CACHE_METHOD) + { + // prometheus status + cacheItemCount++; + if ((pCacheItem->refreshFlag & AUTO_REFRESH_NEVER_TIMEOUT) != 0) + neverTimeoutCacheItemCount++; + else + autoRefreshItemCount++; + + if ((checkFlag == true) && (pCacheItem->refreshFlag & AUTO_REFRESH_UNERASE || pCacheItem->refreshFlag & AUTO_REFRESH_NEVER_TIMEOUT)) + { + //log_debug("[CACHE] detected unerase method(%s) %s", qPrintable(pCacheItem->methodName), itemId.toHex().data()); + continue; + } + qint64 accessDiff = currMTime - pCacheItem->lastAccessTime; + if (accessDiff > accessTimeoutMSeconds) + { + if (pCacheItem->cachedFlag == true || pCacheItem->retryCount > RETRY_RESPONSE_MAX_COUNT) + { + // remove cache item + log_debug("[CACHE] deleting cache item for access timeout %s", itemId.toHex().data()); + deleteIdList.append(itemId); // Safely erase and move to the next item + continue; + } + } + } + else if (pCacheItem->methodType == CacheMethodType::SUBSCRIBE_METHOD) + { + // prometheus status + subscribeItemCount++; + + qint64 refreshDiff = currMTime - pCacheItem->lastRefreshTime; + + if (pCacheItem->clientMap.count() == 0 || refreshDiff > responseTimeoutMSeconds) + { + log_debug("[WS] checking subscription item clientCount=%d diff=%ld", pCacheItem->clientMap.count(), refreshDiff); + + // add unsubscribe request item for cache thread + if (pCacheItem->cachedFlag == true) + { + int ccIndex = get_cc_index_from_clientId(pCacheItem->cacheClientId); + if (ccIndex >= 0) + { + QByteArray instanceId = gWsCacheClientList[ccIndex].instanceId; + UnsubscribeRequestItem reqItem; + reqItem.subscriptionStr = pCacheItem->subscriptionStr; + reqItem.from = pCacheItem->requestPacket.from; + reqItem.unsubscribeMethodName = gSubscribeMethodMap[pCacheItem->methodName]; + reqItem.cacheClientId = pCacheItem->cacheClientId; + gUnsubscribeRequestMap[instanceId].append(reqItem); + } + } + + // remove subscription item + log_debug("[WS] deleting1 subscription item subscriptionStr=\"%s\"", qPrintable(pCacheItem->subscriptionStr)); + deleteIdList.append(itemId); // Safely erase and move to the next item + continue; + } + } + } + + numCacheItem = cacheItemCount; + numSubscriptionItem = subscribeItemCount; + numNeverTimeoutCacheItem = neverTimeoutCacheItemCount; + numAutoRefreshItem = autoRefreshItemCount; + numAREItemCount = cacheItemCount - autoRefreshItemCount; + + for (int i=0; i < deleteIdList.count(); i++) + { + remove_cache_item(deleteIdList[i]); + } + int deleteCount = deleteIdList.count(); + + int totalItemCount = itemCount - deleteCount; + if (totalItemCount < gCacheItemMaxCount) + { + break; + } + + log_debug("[CACHE] detected MAX cache item count %d", totalItemCount); + accessTimeoutMSeconds -= (gAccessTimeoutSeconds/5) * 1000; + + // disable check flag + checkFlag = false; + } +} + +void check_old_clients() +{ + qint64 clientNoRequestTimeoutSeconds = gClientNoRequestTimeoutSeconds * 1000; + qint64 currMTime = QDateTime::currentMSecsSinceEpoch(); + + // lookup clients to delete + foreach(QByteArray id, gHttpClientMap.keys()) + { + qint64 diffMSeconds = currMTime - gHttpClientMap[id].lastRequestTime; + if (!gDeleteClientList.contains(id) && (diffMSeconds > clientNoRequestTimeoutSeconds)) + { + // delete this client + log_debug("[HTTP] add delete client id=%s", id.data()); + gDeleteClientList.append(id); + } + } + + foreach(QByteArray id, gWsClientMap.keys()) + { + qint64 diffMSeconds = currMTime - gWsClientMap[id].lastRequestTime; + if (!gDeleteClientList.contains(id) && (diffMSeconds > clientNoRequestTimeoutSeconds)) + { + // delete this client + log_debug("[WS] add delete client id=%s", id.data()); + gDeleteClientList.append(id); + continue; + } + } + + // count clients + numHttpClientCount = gHttpClientMap.count(); + numWsClientCount = gWsClientMap.count(); +} + +void cache_thread() +{ + gCacheThreadAllowFlag = true; + while (gCacheThreadAllowFlag) + { + while (gMainThreadRunning) + { + gCacheThreadRunning = false; + QThread::usleep(10); + } + gCacheThreadRunning = true; + + remove_old_cache_items(); + check_old_clients(); + + gCacheThreadRunning = false; + + count_clients(); + + QThread::msleep(1000); + } +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Utils +bool is_convertible_to_int(const QString &str) { + bool ok; + str.toInt(&ok); // Attempt conversion to int + return ok; +} + +bool is_cache_method(QString methodStr) +{ + if (gCacheMethodList.contains(methodStr, Qt::CaseInsensitive)) + { + return true; + } + else if (gCacheMethodList.contains("*") && + !gSubscribeMethodMap.contains(methodStr.toLower())) + { + foreach(QString subKey, gSubscribeMethodMap.keys()) + { + if (gSubscribeMethodMap[subKey].toLower() == methodStr.toLower()) + { + return false; + } + } + return true; + } + return false; +} + +bool is_subscribe_method(QString methodStr) +{ + if (gSubscribeMethodMap.contains(methodStr.toLower())) + { + return true; + } + return false; +} + +bool is_never_timeout_method(QString methodStr, QString paramsStr) +{ + if (gNeverTimeoutMethodList.contains(methodStr, Qt::CaseInsensitive)) + { + if (QString::compare(paramsStr, "[LIST]", Qt::CaseInsensitive) != 0) + { + return true; + } + } + return false; +} + +int get_cc_index_from_clientId(QByteArray clientId) +{ + for (int i = 0; i < gWsCacheClientList.count(); i++) + { + if (gWsCacheClientList[i].clientId == clientId) + { + return i; + } + } + return -1; +} + +int get_cc_next_index_from_clientId(QByteArray clientId, QByteArray instanceId) +{ + int ccIndex = get_cc_index_from_clientId(clientId); + + ccIndex += 1; + if (ccIndex >= gWsCacheClientList.count()) + ccIndex = 0; + + for (int i = ccIndex; i < gWsCacheClientList.count(); i++) + { + if (gWsCacheClientList[i].initFlag == true && gWsCacheClientList[i].instanceId == instanceId) + { + return i; + } + } + + for (int i = 0; i < ccIndex; i++) + { + if (gWsCacheClientList[i].initFlag == true && gWsCacheClientList[i].instanceId == instanceId) + { + return i; + } + } + + return -1; +} + +int get_cc_index_from_init_request(ZhttpRequestPacket &p) +{ + QByteArray pId = p.ids.first().id; + + // check if recvInit is from Health_Client or Cache_Client + HttpHeaders requestHeaders = p.headers; + QByteArray headerKey = QByteArray("Socket-Owner"); + // Check Health_Client + if (requestHeaders.contains(headerKey)) + { + QString headerValue = requestHeaders.get(headerKey).data(); + // Define the regular expression to extract the number + QRegularExpression regex("Cache_Client(\\d+)"); + QRegularExpressionMatch match = regex.match(headerValue); + + if (match.hasMatch()) + { + QString numberStr = match.captured(1); + int number = numberStr.toInt(); // Convert to integer + + return number; + } + } + + return -1; +} + +void check_cache_clients() +{ + qint64 currMTime = QDateTime::currentMSecsSinceEpoch(); + for (int i = 0; i < gWsCacheClientList.count(); i++) + { + qint64 diff = currMTime - gWsCacheClientList[i].lastResponseTime; + if (diff > gResponseTimeoutSeconds*1000) + { + log_debug("[WS] detected cache client %d response timeout %s", i, gWsCacheClientList[i].clientId.data()); + gWsCacheClientList[i].initFlag = false; + } + + if (gWsCacheClientList[i].initFlag == false) + { + // Remove all items where value with client id + foreach(QByteArray itemId, gCacheItemMap.keys()) + { + if (gCacheItemMap[itemId].methodType == CacheMethodType::SUBSCRIBE_METHOD && + gCacheItemMap[itemId].cacheClientId == gWsCacheClientList[i].clientId) + { + log_debug("[WS] Remove subscription cache item %s", gCacheItemMap[itemId].subscriptionStr); + remove_cache_item(itemId); + } + } + + log_debug("[WS] killing cache client %d %s", i, gWsCacheClientList[i].clientId.data()); + create_process_for_cacheclient(i); + gWsCacheClientList[i].lastResponseSeq = -1; + gWsCacheClientList[i].lastResponseTime = QDateTime::currentMSecsSinceEpoch(); + } + } + + QTimer::singleShot(30 * 1000, [=]() { + check_cache_clients(); + }); +} + +int get_main_cc_index(QByteArray instanceId) +{ + for (int i=0; i>"+itemKey; + + // add exception for id field + if (itemKey == "id") + { + if (itemVal.type() == QVariant::String) + { + QString strVal = "\""; + strVal += itemVal.toString(); + strVal += "\""; + jsonMap[itemKey] = strVal; + } + else if (itemVal.canConvert()) + { + jsonMap[itemKey] = itemVal.toString(); + } + } + else if (itemVal.canConvert()) + { + jsonMap[itemKey] = itemVal.toString(); + } + else if (itemVal.type() == QVariant::Map) + { + QVariantMap mapData = itemVal.toMap(); + parse_json_map(mapData, itemKey, jsonMap); + } + else if (itemVal.type() == QVariant::List) + { + QString tmpStr = ""; + int i = 0; + for (QVariant m : itemVal.toList()) + { + if (m.canConvert()) + { + tmpStr += m.toString() + "+"; + } + else if (m.type() == QVariant::List) + { + for (QVariant n : m.toList()) + { + if (n.canConvert()) + { + QString s = n.toString(); + if (s.length() == 0) + { + tmpStr += "null"; + tmpStr += "+"; + } + else + { + tmpStr += n.toString() + "+"; + } + } + else + { + log_debug("[WS] invalid type=%s", n.typeName()); + } + } + // remove '+', '/' at the end + while (tmpStr.endsWith("+") || tmpStr.endsWith("/")) + { + tmpStr.remove(tmpStr.length()-1, 1); + } + tmpStr += "/"; + } + else if (m.type() == QVariant::Map) + { + QVariantMap mapData = m.toMap(); + parse_json_map(mapData, itemKey+">>"+QString::number(i), jsonMap); + } + i++; + } + + // remove '+', '/' at the end + while (tmpStr.endsWith("+") || tmpStr.endsWith("/")) + { + tmpStr.remove(tmpStr.length()-1, 1); + } + + jsonMap[itemKey] = (tmpStr.length() > 0) ? tmpStr : "[LIST]"; + } + else + { + log_debug("[WS] unknown parse json type=%s", itemVal.typeName()); + } + } +} + +int parse_json_msg(QVariant jsonMsg, QVariantMap& jsonMap) +{ + // parse body as JSON string + QJsonParseError error; + QJsonDocument jsonDoc = QJsonDocument::fromJson(jsonMsg.toByteArray(), &error); + + if(error.error != QJsonParseError::NoError) + return -1; + + if(jsonDoc.isObject()) + { + QVariantMap jsonData = jsonDoc.object().toVariantMap(); + parse_json_map(jsonData, NULL, jsonMap); + } + else if(jsonDoc.isArray()) + { + QVariantList jsonData = jsonDoc.array().toVariantList(); + for(const QVariant& item : jsonData) + { + if (item.type() == QVariant::Map) + { + QVariantMap mapData = item.toMap(); + parse_json_map(mapData, NULL, jsonMap); + break; + } + } + } + else + { + return -1; + } + + return 0; +} + +int parse_packet_msg(Scheme scheme, const ZhttpRequestPacket& packet, PacketMsg& packetMsg, const QByteArray& instanceId) +{ + // Parse json message + QVariantMap jsonMap; + if (parse_json_msg(packet.toVariant().toHash().value("body"), jsonMap) < 0) + { + log_debug("[JSON] failed to parse json"); + return -1; + } + + for(QVariantMap::const_iterator item = jsonMap.begin(); item != jsonMap.end(); ++item) + { + log_debug("key = %s, value = %s", qPrintable(item.key()), qPrintable(item.value().toString().mid(0,128))); + } + + if (jsonMap.contains(gMsgIdAttrName)) + { + packetMsg.id = jsonMap[gMsgIdAttrName].toString(); + if (jsonMap[gMsgIdAttrName].type() == QVariant::String) + { + packetMsg.id = QString("\"%1\"").arg(packetMsg.id); + } + } + packetMsg.id = jsonMap.contains(gMsgIdAttrName) ? jsonMap[gMsgIdAttrName].toString() : ""; + packetMsg.method = jsonMap.contains(gMsgMethodAttrName) ? jsonMap[gMsgMethodAttrName].toString().toLower() : NULL; + packetMsg.result = jsonMap.contains(gResultAttrName) ? jsonMap[gResultAttrName].toString() : ""; + packetMsg.isResultNull = false; + if (jsonMap.contains(gResultAttrName) && packetMsg.result.isEmpty()) + { + packetMsg.isResultNull = true; + } + else + { + for (auto it = jsonMap.constBegin(); it != jsonMap.constEnd(); ++it) + { + foreach (QString attr, gErrorAttrList) + { + if (it.key().startsWith(attr, Qt::CaseInsensitive)) + { + packetMsg.isResultNull = true; + } + } + } + } + packetMsg.params = jsonMap.contains(gMsgParamsAttrName) ? jsonMap[gMsgParamsAttrName].toString() : ""; + if (scheme == Scheme::http) + { + QString subKey = QString("HTTP+"); + packetMsg.paramsHash = build_hash_key(jsonMap, subKey); + } + else + { + QString subKey = QString("WS+"); + //if (is_subscribe_method(packetMsg.method) == true) + //{ + // subKey += instanceId.constData(); + //} + packetMsg.paramsHash = build_hash_key(jsonMap, subKey); + } + packetMsg.subscription = jsonMap.contains(gSubscriptionAttrName) ? jsonMap[gSubscriptionAttrName].toString() : ""; + packetMsg.resultBlock = jsonMap.contains(gSubscribeBlockAttrName) ? jsonMap[gSubscribeBlockAttrName].toString() : ""; + packetMsg.resultChanges = jsonMap.contains(gSubscribeChangesAttrName) ? jsonMap[gSubscribeChangesAttrName].toString() : ""; + + return 0; +} + +int parse_packet_msg(Scheme scheme, const ZhttpResponsePacket& packet, PacketMsg& packetMsg, const QByteArray& instanceId) +{ + // Parse json message + QVariantMap jsonMap; + if (parse_json_msg(packet.toVariant().toHash().value("body"), jsonMap) < 0) + { + log_debug("[JSON] failed to parse json"); + return -1; + } + + for(QVariantMap::const_iterator item = jsonMap.begin(); item != jsonMap.end(); ++item) + { + log_debug("key = %s, value = %s", qPrintable(item.key()), qPrintable(item.value().toString().mid(0,128))); + } + + if (jsonMap.contains(gMsgIdAttrName)) + { + packetMsg.id = jsonMap[gMsgIdAttrName].toString(); + if (jsonMap[gMsgIdAttrName].type() == QVariant::String) + { + packetMsg.id = QString("\"%1\"").arg(packetMsg.id); + } + } + packetMsg.id = jsonMap.contains(gMsgIdAttrName) ? jsonMap[gMsgIdAttrName].toString() : ""; + packetMsg.method = jsonMap.contains(gMsgMethodAttrName) ? jsonMap[gMsgMethodAttrName].toString().toLower() : NULL; + packetMsg.result = jsonMap.contains(gResultAttrName) ? jsonMap[gResultAttrName].toString() : ""; + packetMsg.isResultNull = false; + if (jsonMap.contains(gResultAttrName) && packetMsg.result.isEmpty()) + { + packetMsg.isResultNull = true; + } + else + { + for (auto it = jsonMap.constBegin(); it != jsonMap.constEnd(); ++it) + { + foreach (QString attr, gErrorAttrList) + { + if (it.key().startsWith(attr, Qt::CaseInsensitive)) + { + packetMsg.isResultNull = true; + } + } + } + } + packetMsg.params = jsonMap.contains(gMsgParamsAttrName) ? jsonMap[gMsgParamsAttrName].toString() : ""; + if (scheme == Scheme::http) + { + QString subKey = QString("HTTP+"); + packetMsg.paramsHash = build_hash_key(jsonMap, subKey); + } + else + { + QString subKey = QString("WS+"); + //if (is_subscribe_method(packetMsg.method) == true) + //{ + // subKey += instanceId.constData(); + //} + packetMsg.paramsHash = build_hash_key(jsonMap, subKey); + } + packetMsg.subscription = jsonMap.contains(gSubscriptionAttrName) ? jsonMap[gSubscriptionAttrName].toString() : ""; + packetMsg.resultBlock = jsonMap.contains(gSubscribeBlockAttrName) ? jsonMap[gSubscribeBlockAttrName].toString() : ""; + packetMsg.resultChanges = jsonMap.contains(gSubscribeChangesAttrName) ? jsonMap[gSubscribeChangesAttrName].toString() : ""; + + return 0; +} + +void replace_id_field(QByteArray &body, QString oldId, int newId) +{ + // new pattern + char newPattern[64]; + qsnprintf(newPattern, 64, "\"id\":%d", newId); + + // find pattern + for (int i = 0; i < 20; i++) + { + QString iSpace = ""; + QString jSpace = ""; + for (int k = 0; k < i; k++) + { + iSpace += " "; + } + for (int j = 0; j < 20; j++) + { + for (int k = 0; k < j; k++) + { + jSpace += " "; + } + QString oldPattern = QString("\"id\"") + iSpace + QString(":") + jSpace + oldId; + int idx = body.indexOf(oldPattern.toLatin1()); + if (idx >= 0) + { + body.replace(idx, oldPattern.length(), newPattern); + return; + } + } + } +} + +void replace_id_field(QByteArray &body, QString oldId, QString newId) +{ + // new pattern + char newPattern[64]; + qsnprintf(newPattern, 64, "\"id\":%s", qPrintable(newId)); + + // find pattern + for (int i = 0; i < 20; i++) + { + QString iSpace = ""; + QString jSpace = ""; + for (int k = 0; k < i; k++) + { + iSpace += " "; + } + for (int j = 0; j < 20; j++) + { + for (int k = 0; k < j; k++) + { + jSpace += " "; + } + QString oldPattern = QString("\"id\"") + iSpace + QString(":") + jSpace + oldId; + int idx = body.indexOf(oldPattern.toLatin1()); + if (idx >= 0) + { + body.replace(idx, oldPattern.length(), newPattern); + return; + } + } + } +} + +void replace_id_field(QByteArray &body, int oldId, QString newId) +{ + // new pattern + char newPattern[64]; + qsnprintf(newPattern, 64, "\"id\":%s", qPrintable(newId)); + + // find pattern + for (int i = 0; i < 20; i++) + { + QString iSpace = ""; + QString jSpace = ""; + for (int k = 0; k < i; k++) + { + iSpace += " "; + } + for (int j = 0; j < 20; j++) + { + for (int k = 0; k < j; k++) + { + jSpace += " "; + } + QString oldPattern = QString("\"id\"") + iSpace + QString(":") + jSpace + QString::number(oldId); + int idx = body.indexOf(oldPattern.toLatin1()); + if (idx >= 0) + { + body.replace(idx, oldPattern.length(), newPattern); + return; + } + } + } +} + +void replace_result_field(QByteArray &body, QString oldResult, QString newResult) +{ + if (oldResult == newResult) + { + return; + } + + QString oldPattern0 = "\"result\":\"" + oldResult + "\""; + QString oldPattern1 = "\"result\": \"" + oldResult + "\""; + + char newPattern0[64], newPattern1[64]; + qsnprintf(newPattern0, 64, "\"result\":\"%s\"", qPrintable(newResult)); + qsnprintf(newPattern1, 64, "\"result\": \"%s\"", qPrintable(newResult)); + + int idx = body.indexOf(oldPattern0.toLatin1()); + if (idx >= 0) + { + body.replace(idx, oldPattern0.length(), newPattern0); + return; + } + + idx = body.indexOf(oldPattern1.toLatin1()); + if (idx >= 0) + { + body.replace(idx, oldPattern1.length(), newPattern1); + } +} + +void replace_subscription_field(QByteArray &body, QString oldSubscription, QString newSubscription) +{ + if (oldSubscription == newSubscription) + { + return; + } + + QString oldPattern0 = "\"subscription\":\"" + oldSubscription + "\""; + QString oldPattern1 = "\"subscription\": \"" + oldSubscription + "\""; + + char newPattern0[64], newPattern1[64]; + qsnprintf(newPattern0, 64, "\"subscription\":\"%s\"", qPrintable(newSubscription)); + qsnprintf(newPattern1, 64, "\"subscription\": \"%s\"", qPrintable(newSubscription)); + + int idx = body.indexOf(oldPattern0.toLatin1()); + if (idx >= 0) + { + body.replace(idx, oldPattern0.length(), newPattern0); + return; + } + + idx = body.indexOf(oldPattern1.toLatin1()); + if (idx >= 0) + { + body.replace(idx, oldPattern1.length(), newPattern1); + } +} + +QByteArray calculate_response_hash_val(QByteArray &responseBody, int idVal) +{ + QByteArray out = responseBody; + // replace id str in response + replace_id_field(out, idVal, 0); + + return QCryptographicHash::hash(out,QCryptographicHash::Sha1); +} + +QByteArray calculate_response_seckey_from_init_request(ZhttpRequestPacket &p) +{ + // parse request packet header + HttpHeaders requestHeaders = p.headers; + if (requestHeaders.contains("Sec-WebSocket-Key")) + { + QByteArray requestKey = requestHeaders.get("Sec-WebSocket-Key"); + QByteArray responseKey = QCryptographicHash::hash((requestKey + MAGIC_STRING), QCryptographicHash::Sha1).toBase64(); + + log_debug("[WS] get ws response key for init request requestKey=%s responseKey=%s", requestKey.data(), responseKey.data()); + + return responseKey; + } + return QByteArray(""); +} + +QByteArray build_hash_key(QVariantMap &jsonMap, QString startingStr) +{ + QString hashKeyStr = startingStr; + for (int i = 0; i < gCacheKeyItemList.count(); i++) + { + CacheKeyItem keyItem = gCacheKeyItemList[i]; + QString keyVal = ""; + if (keyItem.flag == RAW_VALUE) + { + keyVal += keyItem.keyName; + } + else + { + if (keyItem.flag == JSON_PAIR) + { + keyVal += keyItem.keyName + ":"; + } + + for(QVariantMap::const_iterator item = jsonMap.begin(); item != jsonMap.end(); ++item) + { + QString iKey = item.key(); + QString iValue = item.value().toString(); + + if (!iKey.compare(keyItem.keyName, Qt::CaseInsensitive)) + { + if (jsonMap[keyItem.keyName].toString().length() > 0) + keyVal += jsonMap[keyItem.keyName].toString(); + else + keyVal += " "; + } + else if (iKey.indexOf(keyItem.keyName+">>", 0, Qt::CaseInsensitive) == 0) + { + keyVal += iKey.toLower() + "->" + iValue; + } + } + } + if (keyVal.length() > 0) + { + hashKeyStr += keyVal; + if ((i+1) < gCacheKeyItemList.count()) + { + hashKeyStr += "+"; + } + } + } + log_debug("[HASH] Hash-Key-Str = %s", qPrintable(hashKeyStr.mid(0,128))); + + return QCryptographicHash::hash(hashKeyStr.toUtf8(),QCryptographicHash::Sha1); +} + +int check_multi_packets_for_ws_request(ZhttpRequestPacket &p) +{ + QByteArray pId = p.ids.first().id; + // Check if multi-parts request + if (gWsMultiPartRequestItemMap.contains(pId)) + { + // this is middle packet of multi-request + if (p.more == true) + { + log_debug("[WS] Detected middle of multi-parts request"); + gWsMultiPartRequestItemMap[pId].body.append(p.body); + + return -1; + } + else // this is end packet of multi-request + { + log_debug("[WS] Detected end of multi-parts request"); + gWsMultiPartRequestItemMap[pId].body.append(p.body); + p.body = gWsMultiPartRequestItemMap[pId].body; + + gWsMultiPartRequestItemMap.remove(pId); + } + } + else + { + // this is first packet of multi-request + if (p.more == true) + { + log_debug("[WS] Detected start of multi-parts request"); + + // register new multi-request item + gWsMultiPartRequestItemMap[pId] = p; + + // prometheus status + numRequestMultiPart++; + + return -1; + } + } + + return 0; +} + +int check_multi_packets_for_http_response(ZhttpResponsePacket &p) +{ + QByteArray pId = p.ids.first().id; + // Check if multi-parts response + if (gHttpMultiPartResponseItemMap.contains(pId)) + { + // this is middle packet of multi-response + if (p.more == true) + { + log_debug("[HTTP] Detected middle of multi-parts response"); + gHttpMultiPartResponseItemMap[pId].body.append(p.body); + + return -1; + } + else // this is end packet of multi-response + { + log_debug("[HTTP] Detected end of multi-parts response"); + gHttpMultiPartResponseItemMap[pId].body.append(p.body); + p = gHttpMultiPartResponseItemMap[pId]; + + gHttpMultiPartResponseItemMap.remove(pId); + + return 1; + } + } + else + { + // this is first packet of multi-response + if (p.more == true) + { + log_debug("[HTTP] Detected start of multi-parts response"); + + // register new multi-response item + gHttpMultiPartResponseItemMap[pId] = p; + + // prometheus status + numResponseMultiPart++; + + return -1; + } + } + + return 0; +} + +int check_multi_packets_for_ws_response(ZhttpResponsePacket &p) +{ + QByteArray pId = p.ids.first().id; + // Check if multi-parts response + if (gWsMultiPartResponseItemMap.contains(pId)) + { + // this is middle packet of multi-response + if (p.more == true) + { + log_debug("[WS] Detected middle of multi-parts response"); + gWsMultiPartResponseItemMap[pId].body.append(p.body); + + return -1; + } + else // this is end packet of multi-response + { + log_debug("[WS] Detected end of multi-parts response"); + gWsMultiPartResponseItemMap[pId].body.append(p.body); + p.body = gWsMultiPartResponseItemMap[pId].body; + + gWsMultiPartResponseItemMap.remove(pId); + return 1; + } + } + else + { + // this is first packet of multi-response + if (p.more == true) + { + log_debug("[WS] Detected start of multi-parts response"); + + // register new multi-response item + gWsMultiPartResponseItemMap[pId] = p; + + // prometheus status + numResponseMultiPart++; + + return -1; + } + } + + return 0; +} + +int update_request_seq(const QByteArray &clientId) +{ + int ret = -1; + + if (gWsClientMap.contains(clientId)) + { + gWsClientMap[clientId].lastRequestSeq += 1; + ret = gWsClientMap[clientId].lastRequestSeq; + } + else if (gHttpClientMap.contains(clientId)) + { + gHttpClientMap[clientId].lastRequestSeq += 1; + ret = gHttpClientMap[clientId].lastRequestSeq; + } + else // cache client + { + int ccIndex = get_cc_index_from_clientId(clientId); + if (ccIndex >= 0) + { + gWsCacheClientList[ccIndex].lastRequestSeq += 1; + ret = gWsCacheClientList[ccIndex].lastRequestSeq; + } + } + + return ret; +} + +int get_client_new_response_seq(const QByteArray &clientId) +{ + int ret = -1; + if (gWsClientMap.contains(clientId)) + { + ret = gWsClientMap[clientId].lastResponseSeq + 1; + gWsClientMap[clientId].lastResponseSeq = ret; + } + else if (gHttpClientMap.contains(clientId)) + { + ret = gHttpClientMap[clientId].lastResponseSeq + 1; + gHttpClientMap[clientId].lastResponseSeq = ret; + } + else // cache client + { + int ccIndex = get_cc_index_from_clientId(clientId); + if (ccIndex >= 0) + { + ret = gWsCacheClientList[ccIndex].lastResponseSeq + 1; + gWsCacheClientList[ccIndex].lastResponseSeq = ret; + } + } + + return ret; +} + +int get_client_credit_size(const QByteArray &clientId) +{ + int ret = 8192; + if (gWsClientMap.contains(clientId)) + { + ret = gWsClientMap[clientId].creditSize; + } + else if (gHttpClientMap.contains(clientId)) + { + ret = gHttpClientMap[clientId].creditSize; + } + + return ret; +} + +void send_http_post_request_with_refresh_header(QString backend, QByteArray postData, char *headerVal) +{ + // Create the QNetworkAccessManager + QNetworkAccessManager *manager = new QNetworkAccessManager(); + + // Set the target URL + QUrl url(backend); + QNetworkRequest request(url); + + // Set request headers + request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"); + request.setRawHeader(HTTP_REFRESH_HEADER, headerVal); + + // Send the POST request asynchronously + QNetworkReply *reply = manager->post(request, postData); + /* + // Ignore the response - don't connect any slots to 'reply->finished' + QObject::connect(reply, &QNetworkReply::finished, reply, &QNetworkReply::deleteLater); + + // Optionally, delete manager after sending request (if you don't need it later) + QObject::connect(reply, &QNetworkReply::destroyed, manager, &QNetworkAccessManager::deleteLater); + */ + // Disconnect immediately without waiting for a reply + QObject::connect(reply, &QNetworkReply::finished, [reply]() { + reply->deleteLater(); // Clean up reply object + }); + + // Optionally, delete manager after request is sent + QObject::connect(reply, &QNetworkReply::finished, manager, &QNetworkAccessManager::deleteLater); +} + +int get_next_cache_refresh_interval(const QByteArray &itemId) +{ + int timeInterval = 0; + + CacheItem *pCacheItem = load_cache_item(itemId); + if (pCacheItem == NULL) + { + log_debug("[CACHE] not exist cache item %s", itemId.toHex().data()); + return -1; + } + + if (pCacheItem->cachedFlag == true) + { + // if it`s websocket and cache method + if (pCacheItem->proto == Scheme::http || + (pCacheItem->proto == Scheme::websocket && pCacheItem->methodType == CacheMethodType::CACHE_METHOD)) + { + if (pCacheItem->refreshFlag & AUTO_REFRESH_NEVER_TIMEOUT) + { + timeInterval = 0; + } + else if (pCacheItem->refreshFlag & AUTO_REFRESH_SHORTER_TIMEOUT) + { + timeInterval = gShorterTimeoutSeconds; + } + else if (pCacheItem->refreshFlag & AUTO_REFRESH_LONGER_TIMEOUT) + { + timeInterval = gLongerTimeoutSeconds; + } + else + { + timeInterval = gCacheTimeoutSeconds; + } + } + } + else + { + // set interval to the fixed value + timeInterval = gBackendSwitchIntervalSeconds; + } + + return timeInterval; +} + +QString get_switched_http_backend_url(QString currUrl) +{ + int index = -1; + + // Iterate through the list + for (int i = 0; i < gHttpBackendUrlList.size(); ++i) + { + if (gHttpBackendUrlList.at(i).compare(currUrl, Qt::CaseInsensitive) == 0) + { + index = i; + break; // Stop after finding the match + } + } + + // Check the result + if (index != -1) + { + index++; + if (index >= gHttpBackendUrlList.size()) + index = 0; + } + else + { + index = 0; + } + + return gHttpBackendUrlList[index]; +} + +QString get_switched_ws_backend_url(QString currUrl) +{ + int index = -1; + + // Iterate through the list + for (int i = 0; i < gWsBackendUrlList.size(); ++i) + { + if (gWsBackendUrlList.at(i).compare(currUrl, Qt::CaseInsensitive) == 0) + { + index = i; + break; // Stop after finding the match + } + } + + // Check the result + if (index != -1) + { + index++; + if (index >= gWsBackendUrlList.size()) + index = 0; + } + else + { + index = 0; + } + + return gWsBackendUrlList[index]; +} + +void count_requests(QString methodName) +{ + // count methods + numRequestReceived++; + if (methodName.indexOf("author_", 0, Qt::CaseInsensitive) == 0) { + numRpcAuthor++; + if (methodName.indexOf("submitandwatchextrinsic", 0, Qt::CaseInsensitive) == 7) + numRpcSubscribe++; + } else if (methodName.indexOf("babe_", 0, Qt::CaseInsensitive) == 0) { + numRpcBabe++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 5) + numRpcSubscribe++; + } else if (methodName.indexOf("beefy_", 0, Qt::CaseInsensitive) == 0) { + numRpcBeefy++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 6) + numRpcSubscribe++; + } else if (methodName.indexOf("chain_", 0, Qt::CaseInsensitive) == 0) { + numRpcChain++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 6) + numRpcSubscribe++; + } else if (methodName.indexOf("childstate_", 0, Qt::CaseInsensitive) == 0) { + numRpcChildState++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 11) + numRpcSubscribe++; + } else if (methodName.indexOf("contracts_", 0, Qt::CaseInsensitive) == 0) { + numRpcContracts++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 10) + numRpcSubscribe++; + } else if (methodName.indexOf("dev_", 0, Qt::CaseInsensitive) == 0) { + numRpcDev++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 4) + numRpcSubscribe++; + } else if (methodName.indexOf("engine_", 0, Qt::CaseInsensitive) == 0) { + numRpcEngine++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 7) + numRpcSubscribe++; + } else if (methodName.indexOf("eth_", 0, Qt::CaseInsensitive) == 0) { + numRpcEth++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 4) + numRpcSubscribe++; + } else if (methodName.indexOf("net_", 0, Qt::CaseInsensitive) == 0) { + numRpcNet++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 4) + numRpcSubscribe++; + } else if (methodName.indexOf("web3_", 0, Qt::CaseInsensitive) == 0) { + numRpcWeb3++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 5) + numRpcSubscribe++; + } else if (methodName.indexOf("grandpa_", 0, Qt::CaseInsensitive) == 0) { + numRpcGrandpa++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 8) + numRpcSubscribe++; + } else if (methodName.indexOf("mmr_", 0, Qt::CaseInsensitive) == 0) { + numRpcMmr++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 4) + numRpcSubscribe++; + } else if (methodName.indexOf("offchain_", 0, Qt::CaseInsensitive) == 0) { + numRpcOffchain++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 9) + numRpcSubscribe++; + } else if (methodName.indexOf("payment_", 0, Qt::CaseInsensitive) == 0) { + numRpcPayment++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 8) + numRpcSubscribe++; + } else if (methodName.indexOf("rpc_", 0, Qt::CaseInsensitive) == 0) { + numRpcRpc++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 4) + numRpcSubscribe++; + } else if (methodName.indexOf("state_", 0, Qt::CaseInsensitive) == 0) { + numRpcState++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 6) + numRpcSubscribe++; + } else if (methodName.indexOf("sync_state_", 0, Qt::CaseInsensitive) == 0) { + numRpcSyncstate++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 11) + numRpcSubscribe++; + } else if (methodName.indexOf("system_", 0, Qt::CaseInsensitive) == 0) { + numRpcSystem++; + if (methodName.indexOf("subscribe", 0, Qt::CaseInsensitive) == 7) + numRpcSubscribe++; + } + + // add ws Cache lookup count + if (is_cache_method(methodName)) + { + numCacheLookup++; + } + else if (is_subscribe_method(methodName)) + { + numSubscriptionLookup++; + } + + // user-defined method group count + foreach(QString groupKey, gCountMethodGroupMap.keys()) + { + QStringList groupStrList = gCountMethodGroupMap[groupKey]; + + if (groupStrList.contains(methodName, Qt::CaseInsensitive)) + { + groupMethodCountMap[groupKey]++; + } + } +} + +void count_responses(QString responseKey) +{ + // count methods + if (responseKey == "HTTP" || responseKey == "WS") + numMessageSent++; + else if (responseKey == "WS_INIT") + numWsConnect++; +} + +void count_clients() +{ + // client count + numHttpClientCount = gHttpClientMap.count(); + numWsClientCount = gWsClientMap.count(); + numClientCount = numHttpClientCount + numWsClientCount; +} + +void update_prometheus_hit_count(const CacheItem &cacheItem) +{ + // prometheus status + if (cacheItem.methodType == CACHE_METHOD) + { + if ((cacheItem.refreshFlag & AUTO_REFRESH_NEVER_TIMEOUT) != 0) + numNeverTimeoutCacheHit++; + else + numCacheHit++; + } + else if (cacheItem.methodType == SUBSCRIBE_METHOD) + { + numSubscriptionHit++; + } +} + +void restore_prometheusStatFromFile() +{ + QString prometheusBackupFile = QDir::cleanPath(gPrometheusBackupDir + "/prometheus_pushpin.bin"); + + if(prometheusBackupFile.isEmpty()) + return; + + // get last modified time + QFileInfo fileInfo(prometheusBackupFile); + int diffTime = 0xFFFFFF; + int restoreAllowSeconds = gPrometheusRestoreAllowSeconds; + if (fileInfo.exists()) + { + QDateTime lastModifiedTime = fileInfo.lastModified(); + diffTime = (int)(lastModifiedTime.msecsTo(QDateTime::currentDateTime())/1000); + } + + char fName[256]; + sprintf(fName, "%s", qPrintable(prometheusBackupFile)); + + log_info("filename=%s, diffTime=%d, confSeconds=%d", fName, diffTime, restoreAllowSeconds); + if (diffTime < restoreAllowSeconds) + { + FILE *in = fopen(fName, "rb"); + if (in) + { + size_t readLen = 0; + readLen += fread(&numRequestReceived, sizeof(unsigned long long), 1, in); + readLen += fread(&numMessageSent, sizeof(unsigned long long), 1, in); + readLen += fread(&numWsConnect, sizeof(unsigned long long), 1, in); + readLen += fread(&numClientCount, sizeof(unsigned long long), 1, in); + readLen += fread(&numHttpClientCount, sizeof(unsigned long long), 1, in); + readLen += fread(&numWsClientCount, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcAuthor, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcBabe, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcBeefy, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcChain, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcChildState, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcContracts, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcDev, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcEngine, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcEth, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcNet, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcWeb3, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcGrandpa, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcMmr, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcOffchain, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcPayment, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcRpc, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcState, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcSyncstate, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcSystem, sizeof(unsigned long long), 1, in); + readLen += fread(&numRpcSubscribe, sizeof(unsigned long long), 1, in); + readLen += fread(&numCacheInsert, sizeof(unsigned long long), 1, in); + readLen += fread(&numCacheHit, sizeof(unsigned long long), 1, in); + readLen += fread(&numNeverTimeoutCacheInsert, sizeof(unsigned long long), 1, in); + readLen += fread(&numNeverTimeoutCacheHit, sizeof(unsigned long long), 1, in); + readLen += fread(&numCacheLookup, sizeof(unsigned long long), 1, in); + readLen += fread(&numCacheExpiry, sizeof(unsigned long long), 1, in); + readLen += fread(&numRequestMultiPart, sizeof(unsigned long long), 1, in); + readLen += fread(&numSubscriptionInsert, sizeof(unsigned long long), 1, in); + readLen += fread(&numSubscriptionHit, sizeof(unsigned long long), 1, in); + readLen += fread(&numSubscriptionLookup, sizeof(unsigned long long), 1, in); + readLen += fread(&numSubscriptionExpiry, sizeof(unsigned long long), 1, in); + readLen += fread(&numResponseMultiPart, sizeof(unsigned long long), 1, in); + + // user-defined method group count + foreach(QString groupKey, groupMethodCountMap.keys()) + { + readLen += fread(&groupMethodCountMap[groupKey], sizeof(unsigned long long), 1, in); + } + + log_info("requestReceived = %d, restoreItemCount=%d", numRequestReceived, readLen); + + fclose(in); + } + } +} + +void save_prometheusStatIntoFile() +{ + // backup prometheus stat + QString prometheusBackupFile = QDir::cleanPath(gPrometheusBackupDir + "/prometheus_pushpin.bin"); + + char fName[256]; + sprintf(fName, "%s", qPrintable(prometheusBackupFile)); + + log_info(fName); + if (fName) + { + log_info("requestReceived = %d", numRequestReceived); + + FILE *out = fopen(fName, "wb"); + if (out) + { + fwrite(&numRequestReceived, sizeof(unsigned long long), 1, out); + fwrite(&numMessageSent, sizeof(unsigned long long), 1, out); + fwrite(&numWsConnect, sizeof(unsigned long long), 1, out); + fwrite(&numClientCount, sizeof(unsigned long long), 1, out); + fwrite(&numHttpClientCount, sizeof(unsigned long long), 1, out); + fwrite(&numWsClientCount, sizeof(unsigned long long), 1, out); + fwrite(&numRpcAuthor, sizeof(unsigned long long), 1, out); + fwrite(&numRpcBabe, sizeof(unsigned long long), 1, out); + fwrite(&numRpcBeefy, sizeof(unsigned long long), 1, out); + fwrite(&numRpcChain, sizeof(unsigned long long), 1, out); + fwrite(&numRpcChildState, sizeof(unsigned long long), 1, out); + fwrite(&numRpcContracts, sizeof(unsigned long long), 1, out); + fwrite(&numRpcDev, sizeof(unsigned long long), 1, out); + fwrite(&numRpcEngine, sizeof(unsigned long long), 1, out); + fwrite(&numRpcEth, sizeof(unsigned long long), 1, out); + fwrite(&numRpcNet, sizeof(unsigned long long), 1, out); + fwrite(&numRpcWeb3, sizeof(unsigned long long), 1, out); + fwrite(&numRpcGrandpa, sizeof(unsigned long long), 1, out); + fwrite(&numRpcMmr, sizeof(unsigned long long), 1, out); + fwrite(&numRpcOffchain, sizeof(unsigned long long), 1, out); + fwrite(&numRpcPayment, sizeof(unsigned long long), 1, out); + fwrite(&numRpcRpc, sizeof(unsigned long long), 1, out); + fwrite(&numRpcState, sizeof(unsigned long long), 1, out); + fwrite(&numRpcSyncstate, sizeof(unsigned long long), 1, out); + fwrite(&numRpcSystem, sizeof(unsigned long long), 1, out); + fwrite(&numRpcSubscribe, sizeof(unsigned long long), 1, out); + fwrite(&numCacheInsert, sizeof(unsigned long long), 1, out); + fwrite(&numCacheHit, sizeof(unsigned long long), 1, out); + fwrite(&numNeverTimeoutCacheInsert, sizeof(unsigned long long), 1, out); + fwrite(&numNeverTimeoutCacheHit, sizeof(unsigned long long), 1, out); + fwrite(&numCacheLookup, sizeof(unsigned long long), 1, out); + fwrite(&numCacheExpiry, sizeof(unsigned long long), 1, out); + fwrite(&numRequestMultiPart, sizeof(unsigned long long), 1, out); + fwrite(&numSubscriptionInsert, sizeof(unsigned long long), 1, out); + fwrite(&numSubscriptionHit, sizeof(unsigned long long), 1, out); + fwrite(&numSubscriptionLookup, sizeof(unsigned long long), 1, out); + fwrite(&numSubscriptionExpiry, sizeof(unsigned long long), 1, out); + fwrite(&numResponseMultiPart, sizeof(unsigned long long), 1, out); + + // user-defined method group count + foreach(QString groupKey, groupMethodCountMap.keys()) + { + fwrite(&groupMethodCountMap[groupKey], sizeof(unsigned long long), 1, out); + } + + fclose(out); + } + } +} + +pid_t create_process_for_cacheclient_(QString urlPath, int _no) +{ + int master_fd; + pid_t processId = forkpty(&master_fd, NULL, NULL, NULL); + //pid_t processId = fork(); + if (processId == -1) + { + // processId == -1 means error occurred + log_debug("can't fork to start wscat"); + return -1; + } + else if (processId == 0) // child process + { + /* + char *bin = (char*)"/usr/bin/wscat"; + char *envp[] = {NULL}; + + // create wscat + char * argv_list[] = { + bin, + (char*)"-H", socketHeaderStr, + (char*)"-c", (char*)qPrintable(urlPath), + NULL + }; + log_debug("%s %s %s", bin, socketHeaderStr, (char*)qPrintable(urlPath)); + execve(bin, argv_list, envp); + */ + + QString cmdStr = "wscat -H Socket-Owner:Cache_Client"; + cmdStr += QString::number(_no); + cmdStr += " -c "; + cmdStr += urlPath; + log_debug("%s", cmdStr.toUtf8().data()); + char *args[] = {(char *)"bash", (char *)"-c", cmdStr.toUtf8().data(), NULL}; + execvp("bash", args); + + //set_debugLogLevel(true); + log_debug("failed to start wscat error=%d", errno); + + exit(0); + } + + // parent process + log_debug("[WS] created new cache client%d parent=%d processId=%d", _no, getpid(), processId); + + return processId; +} + +pid_t create_process_for_cacheclient__(QString urlPath, int _no) +{ + WebSocketWorker* worker = new WebSocketWorker; + worker->url = urlPath; + + QString headerStr = "Socket-Owner:Cache_Client"; + headerStr += QString::number(_no); + worker->headers << headerStr; + worker->start(); + + log_debug("[WS] started cache client %d", _no); + + return _no+1; +} + +void create_process_for_cacheclient(int _no) +{ + exit_process_for_cacheclient(_no); + + QThread *thread = new QThread; + + WscatWorker *worker = new WscatWorker; + + worker->moveToThread(thread); + + QString headerStr = "Socket-Owner:Cache_Client"; + headerStr += QString::number(_no); + QObject::connect(thread, &QThread::started, [=]() { + QStringList headers; + headers << headerStr; + worker->startWscat(gWsCacheClientList[_no].urlPath, headers); + }); + + thread->start(); + + log_debug("[WS] started cache client %d", _no); + + gWsCacheClientList[_no].wscatWorker = worker; + gWsCacheClientList[_no].wscatThread = thread; +} + +void exit_process_for_cacheclient(int _no) +{ + if (gWsCacheClientList[_no].wscatWorker != nullptr) + { + gWsCacheClientList[_no].wscatWorker->stopWscat(); + gWsCacheClientList[_no].wscatWorker = nullptr; + } + if (gWsCacheClientList[_no].wscatThread != nullptr) + { + gWsCacheClientList[_no].wscatThread->quit(); + gWsCacheClientList[_no].wscatThread->wait(); + gWsCacheClientList[_no].wscatThread = nullptr; + } +} diff --git a/src/core/cacheutil.h b/src/core/cacheutil.h new file mode 100644 index 000000000..1444ee0ce --- /dev/null +++ b/src/core/cacheutil.h @@ -0,0 +1,258 @@ +/* + * Copyright (C) 2017 Fanout, Inc. + * Copyright (C) 2024 Fastly, Inc. + * + * This file is part of Pushpin. + * + * $FANOUT_BEGIN_LICENSE:APACHE2$ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $FANOUT_END_LICENSE$ + */ + +#ifndef CACHEUTIL_H +#define CACHEUTIL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zhttprequestpacket.h" +#include "zhttpresponsepacket.h" +#include "zwebsocket.h" +#include "log.h" +#include "packet/httprequestdata.h" +#include "packet/httpresponsedata.h" + +#include "redispool.h" +#include "wscatworker.h" + +#define AUTO_REFRESH_SHORTER_TIMEOUT 0x01 +#define AUTO_REFRESH_LONGER_TIMEOUT 0x02 +#define AUTO_REFRESH_NEVER_TIMEOUT 0x04 +#define AUTO_REFRESH_UNERASE 0x08 +#define AUTO_REFRESH_EXCLUDE 0x10 +#define AUTO_REFRESH_PASSTHROUGH 0x20 +#define ACCEPT_NULL_RESPONSE 0x40 + +#define HTTP_REFRESH_HEADER "HTTP_REFRESH_REQUEST" + +#define RESPONSE_ID_MARK 9999999 + +#define RETRY_RESPONSE_MAX_COUNT 5 + +enum Scheme { + none, + http, + websocket +}; + +// cache client params +struct ClientItem { + QString urlPath; + WscatWorker *wscatWorker; + QThread *wscatThread; + bool initFlag; + int creditSize; + QString resultStr; + int msgIdCount; + int lastRequestSeq; + int lastResponseSeq; + qint64 lastRequestTime; + qint64 lastResponseTime; + QByteArray receiver; + QByteArray from; + QByteArray clientId; + QByteArray instanceId; +}; + +// Cache key item +enum ItemFlag { + JSON_VALUE, + JSON_PAIR, + RAW_VALUE +}; +struct CacheKeyItem { + QString keyName; + ItemFlag flag; +}; +enum CacheMethodType { + CACHE_METHOD, + SUBSCRIBE_METHOD +}; + +struct ClientInCacheItem { + QString msgId; + QByteArray from; + QByteArray instanceId; +}; + +// Cache Item +struct CacheItem { + int newMsgId; + char refreshFlag; + qint64 lastRequestTime; + qint64 lastRefreshTime; + qint64 lastRefreshCount; + qint64 lastAccessTime; + bool cachedFlag; + Scheme proto; + int retryCount; + int httpBackendNo; + QByteArray cacheClientId; + QString methodName; + ZhttpRequestPacket requestPacket; + CacheMethodType methodType; + QString subscriptionStr; + int subscriptionUpdateCount; + QHash clientMap; + QString blockStr; + QHash changesMap; +}; + +struct UnsubscribeRequestItem { + QString subscriptionStr; + QByteArray from; + QString unsubscribeMethodName; + QByteArray cacheClientId; +}; + +struct PacketMsg { + QString id; + QString method; + QString result; + bool isResultNull; + QString params; + QByteArray paramsHash; + QString subscription; + QString resultBlock; + QString resultChanges; +}; + +void pause_cache_thread(); +void resume_cache_thread(); +void cache_thread(); + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Redis +void redis_removeall_cache_item(); +void redis_reset_replica(); + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Cache Item +bool is_cache_item(const QByteArray& itemId, QString methodName=""); +CacheItem* load_cache_item(const QByteArray& itemId, QString methodName=""); +void store_cache_item_field(const QByteArray& itemId, const char* fieldName, const int& value); +void store_cache_item_field(const QByteArray& itemId, const char* fieldName, const QByteArray& value); +void store_cache_item_field(const QByteArray& itemId, const char* fieldName, const QString& value); +void store_cache_item_field(const QByteArray& itemId, const char* fieldName, const qint64& value); +void store_cache_item_field(const QByteArray& itemId, const char* fieldName, const QHash& value); +void create_cache_item(const QByteArray& itemId, const CacheItem& cacheItem); +void remove_cache_item(const QByteArray& itemId); +QList get_cache_item_ids(); + +void redis_remove_item(const QByteArray& itemId); +void redis_store_cache_response(const QByteArray& itemId, const QByteArray& response); +QByteArray redis_load_cache_response(const QByteArray& itemId); +void store_cache_response_buffer(const QByteArray& itemId, const QByteArray& responseBuf, QString msgId, int addLen); +QByteArray load_cache_response_buffer(const QByteArray& instanceAddress, const QByteArray& itemId, QByteArray packetId, int seqNum, QString msgId, QByteArray from, int addLen); + +bool is_convertible_to_int(const QString &str); +bool is_cache_method(QString methodStr); +bool is_subscribe_method(QString methodStr); +bool is_never_timeout_method(QString methodStr, QString paramsStr); + +void create_process_for_cacheclient(int _no); +void exit_process_for_cacheclient(int _no); + +int get_main_cc_index(QByteArray instanceId); +int get_cc_index_from_clientId(QByteArray clientId); +int get_cc_index_from_init_request(ZhttpRequestPacket &p); +int get_cc_next_index_from_clientId(QByteArray clientId, QByteArray instanceId); + +int parse_json_msg(QVariant jsonMsg, QVariantMap& jsonMap); +int parse_packet_msg(Scheme scheme, const ZhttpRequestPacket& packet, PacketMsg& packetMsg, const QByteArray& instanceId); +int parse_packet_msg(Scheme scheme, const ZhttpResponsePacket& packet, PacketMsg& packetMsg, const QByteArray& instanceId); + +void replace_id_field(QByteArray &body, QString oldId, int newId); +void replace_id_field(QByteArray &body, QString oldId, QString newId); +void replace_id_field(QByteArray &body, int oldId, QString newId); +void replace_result_field(QByteArray &body, QString oldResult, QString newResult); +void replace_subscription_field(QByteArray &body, QString oldSubscription, QString newSubscription); + +QByteArray calculate_response_hash_val(QByteArray &responseBody, int idVal); +QByteArray calculate_response_seckey_from_init_request(ZhttpRequestPacket &p); + +QByteArray build_hash_key(QVariantMap &jsonMap, QString startingStr); + +int check_multi_packets_for_http_response(ZhttpResponsePacket &p); +int check_multi_packets_for_ws_request(ZhttpRequestPacket &p); +int check_multi_packets_for_ws_response(ZhttpResponsePacket &p); + +int update_request_seq(const QByteArray &clientId); +int get_client_new_response_seq(const QByteArray &clientId); +int get_client_credit_size(const QByteArray &clientId); + +void send_http_post_request_with_refresh_header(QString backend, QByteArray postData, char *headerVal); + +int get_next_cache_refresh_interval(const QByteArray &itemId); + +QString get_switched_http_backend_url(QString currUrl); +QString get_switched_ws_backend_url(QString currUrl); + +void check_cache_clients(); + +void count_requests(QString methodName); +void count_responses(QString responseKey); +void count_clients(); + +void update_prometheus_hit_count(const CacheItem &cacheItem); + +void restore_prometheusStatFromFile(); +void save_prometheusStatIntoFile(); + +class WebSocketWorker : public QThread { + Q_OBJECT +public: + QString url; + QStringList headers; + + void run() override + { + QThread::msleep(100); + QProcess process; + + QStringList args; + for (const QString& h : headers) { + args << "--header" << h; + } + args << "-c" << url; + + process.start("wscat", args); + if (!process.waitForStarted()) { + log_debug("Failed to start wscat"); + return; + } + + process.waitForFinished(-1); // Wait until wscat exits + } +}; + +#endif diff --git a/src/core/core.pri b/src/core/core.pri index 796083b63..89096a66f 100644 --- a/src/core/core.pri +++ b/src/core/core.pri @@ -1,4 +1,15 @@ -include(qzmq/src/src.pri) +HEADERS += \ + $$PWD/qzmqcontext.h \ + $$PWD/qzmqsocket.h \ + $$PWD/qzmqvalve.h \ + $$PWD/qzmqreqmessage.h \ + $$PWD/qzmqreprouter.h + +SOURCES += \ + $$PWD/qzmqcontext.cpp \ + $$PWD/qzmqsocket.cpp \ + $$PWD/qzmqvalve.cpp \ + $$PWD/qzmqreprouter.cpp HEADERS += $$PWD/processquit.h SOURCES += $$PWD/processquit.cpp @@ -42,7 +53,13 @@ HEADERS += \ $$PWD/config.h \ $$PWD/timerwheel.h \ $$PWD/jwt.h \ - $$PWD/rtimer.h \ + $$PWD/timer.h \ + $$PWD/defercall.h \ + $$PWD/socketnotifier.h \ + $$PWD/eventloop.h \ + $$PWD/wscatworker.h \ + $$PWD/cacheutil.h \ + $$PWD/redispool.h \ $$PWD/logutil.h \ $$PWD/uuidutil.h \ $$PWD/zutil.h \ @@ -65,7 +82,13 @@ SOURCES += \ $$PWD/config.cpp \ $$PWD/timerwheel.cpp \ $$PWD/jwt.cpp \ - $$PWD/rtimer.cpp \ + $$PWD/timer.cpp \ + $$PWD/defercall.cpp \ + $$PWD/socketnotifier.cpp \ + $$PWD/eventloop.cpp \ + $$PWD/wscatworker.cpp \ + $$PWD/cacheutil.cpp \ + $$PWD/redispool.cpp \ $$PWD/logutil.cpp \ $$PWD/uuidutil.cpp \ $$PWD/zutil.cpp \ diff --git a/src/core/coretests.h b/src/core/coretests.h index 89ccc6715..0bcd76b0e 100644 --- a/src/core/coretests.h +++ b/src/core/coretests.h @@ -3,5 +3,6 @@ int httpheaders_test(int argc, char **argv); int jwt_test(int argc, char **argv); +int eventloop_test(int argc, char **argv); #endif diff --git a/src/core/defercall.cpp b/src/core/defercall.cpp new file mode 100644 index 000000000..95c257ff2 --- /dev/null +++ b/src/core/defercall.cpp @@ -0,0 +1,65 @@ +/* +* Copyright (C) 2025 Fastly, Inc. +* +* This file is part of Pushpin. +* +* $FANOUT_BEGIN_LICENSE:APACHE2$ +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* $FANOUT_END_LICENSE$ +*/ + +#include "defercall.h" +#include + +static thread_local DeferCall *g_instance = nullptr; + +DeferCall::DeferCall() = default; + +DeferCall::~DeferCall() = default; + +void DeferCall::defer(std::function handler) +{ + Call c; + c.handler = handler; + + deferredCalls_.push_back(c); + + QMetaObject::invokeMethod(this, "callNext", Qt::QueuedConnection); +} + +DeferCall *DeferCall::global() +{ + if(!g_instance) + g_instance = new DeferCall; + + return g_instance; +} + +void DeferCall::cleanup() +{ + delete g_instance; + g_instance = nullptr; +} + +void DeferCall::callNext() +{ + // there can't be more invokeMethod resolutions than queued calls + assert(!deferredCalls_.empty()); + + Call c = deferredCalls_.front(); + deferredCalls_.pop_front(); + + c.handler(); +} diff --git a/src/core/defercall.h b/src/core/defercall.h new file mode 100644 index 000000000..e4c76eef8 --- /dev/null +++ b/src/core/defercall.h @@ -0,0 +1,67 @@ +/* +* Copyright (C) 2025 Fastly, Inc. +* +* This file is part of Pushpin. +* +* $FANOUT_BEGIN_LICENSE:APACHE2$ +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* $FANOUT_END_LICENSE$ +*/ + +#ifndef DEFERCALL_H +#define DEFERCALL_H + +#include + +// queues calls to be run after returning to the event loop +class DeferCall : public QObject +{ + Q_OBJECT + +public: + DeferCall(); + ~DeferCall(); + + // queue handler to be called after returning to the event loop. if + // handler contains references, they must outlive DeferCall. the + // recommended usage is for each object needing to perform deferred calls + // to keep a DeferCall as a member variable, and only refer to the + // object's own data in the handler. that way, any references are + // guaranteed to live long enough. + void defer(std::function handler); + + static DeferCall *global(); + static void cleanup(); + + template + static void deleteLater(T *p) + { + global()->defer([=] { delete p; }); + } + +private slots: + void callNext(); + +private: + class Call + { + public: + std::function handler; + }; + + std::list deferredCalls_; +}; + +#endif diff --git a/src/core/eventloop.cpp b/src/core/eventloop.cpp new file mode 100644 index 000000000..277cce56f --- /dev/null +++ b/src/core/eventloop.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2025 Fastly, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eventloop.h" + +#include + +static thread_local EventLoop *g_instance = nullptr; + +EventLoop::EventLoop(int capacity) +{ + // only one per thread allowed + assert(!g_instance); + + inner_ = ffi::event_loop_create(capacity); + + g_instance = this; +} + +EventLoop::~EventLoop() +{ + ffi::event_loop_destroy(inner_); + + g_instance = nullptr; +} + +int EventLoop::exec() +{ + return ffi::event_loop_exec(inner_); +} + +void EventLoop::exit(int code) +{ + ffi::event_loop_exit(inner_, code); +} + +int EventLoop::registerFd(int fd, unsigned char interest, void (*cb)(void *), void *ctx) +{ + size_t id; + + if(ffi::event_loop_register_fd(inner_, fd, interest, cb, ctx, &id) != 0) + return -1; + + return (int)id; +} + +int EventLoop::registerTimer(int timeout, void (*cb)(void *), void *ctx) +{ + size_t id; + + if(ffi::event_loop_register_timer(inner_, timeout, cb, ctx, &id) != 0) + return -1; + + return (int)id; +} + +void EventLoop::deregister(int id) +{ + ffi::event_loop_deregister(inner_, id); +} + +EventLoop *EventLoop::instance() +{ + return g_instance; +} diff --git a/src/core/eventloop.h b/src/core/eventloop.h new file mode 100644 index 000000000..3a02e2bb9 --- /dev/null +++ b/src/core/eventloop.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2025 Fastly, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EVENTLOOP_H +#define EVENTLOOP_H + +#include "rust/bindings.h" + +class EventLoop +{ +public: + enum Interest + { + Readable = ffi::READABLE, + Writable = ffi::WRITABLE, + }; + + EventLoop(int capacity); + ~EventLoop(); + + int exec(); + void exit(int code); + + int registerFd(int fd, unsigned char interest, void (*cb)(void *), void *ctx); + int registerTimer(int timeout, void (*cb)(void *), void *ctx); + void deregister(int id); + + static EventLoop *instance(); + +private: + ffi::EventLoopRaw *inner_; +}; + +#endif diff --git a/src/core/eventloop.rs b/src/core/eventloop.rs new file mode 100644 index 000000000..88d5880dc --- /dev/null +++ b/src/core/eventloop.rs @@ -0,0 +1,639 @@ +/* + * Copyright (C) 2025 Fastly, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::core::list; +use crate::core::reactor; +use crate::core::waker; +use slab::Slab; +use std::cell::{Cell, RefCell}; +use std::future::Future; +use std::os::fd::RawFd; +use std::pin::Pin; +use std::rc::{Rc, Weak}; +use std::task::{Context, Poll, Waker}; +use std::time::Duration; + +pub const READABLE: u8 = 0x01; +pub const WRITABLE: u8 = 0x02; + +pub trait Callback { + fn call(&mut self); +} + +impl Callback for Box { + fn call(&mut self) { + (**self).call(); + } +} + +pub struct FnCallback(T); + +impl Callback for FnCallback { + fn call(&mut self) { + self.0(); + } +} + +enum Evented { + Fd(reactor::FdEvented), + Timer(reactor::TimerEvented), +} + +impl Evented { + fn registration(&self) -> &reactor::Registration { + match self { + Self::Fd(e) => e.registration(), + Self::Timer(e) => e.registration(), + } + } +} + +struct Registration { + _evented: Evented, + activated: bool, + callback: Option, +} + +struct RegistrationsData { + nodes: Slab>>, + activated: list::List, + waker: Option, +} + +#[derive(Debug)] +struct RegistrationsError; + +struct Registrations { + data: RefCell>, +} + +impl Registrations { + fn new(capacity: usize) -> Self { + Self { + data: RefCell::new(RegistrationsData { + nodes: Slab::with_capacity(capacity), + activated: list::List::default(), + waker: None, + }), + } + } + + fn add( + &self, + evented: Evented, + interest: mio::Interest, + get_waker: W, + callback: C, + ) -> Result + where + W: FnOnce(usize) -> Waker, + C: Callback, + { + let data = &mut *self.data.borrow_mut(); + + if data.nodes.len() == data.nodes.capacity() { + return Err(RegistrationsError); + } + + let entry = data.nodes.vacant_entry(); + let nkey = entry.key(); + + evented.registration().set_waker(&get_waker(nkey), interest); + + let reg = Registration { + _evented: evented, + activated: false, + callback: Some(callback), + }; + + entry.insert(list::Node::new(reg)); + + Ok(nkey) + } + + fn remove(&self, reg_id: usize) { + let nkey = reg_id; + + let data = &mut *self.data.borrow_mut(); + + data.activated.remove(&mut data.nodes, nkey); + data.nodes.remove(nkey); + } + + fn activate(&self, reg_id: usize) { + let nkey = reg_id; + + let data = &mut *self.data.borrow_mut(); + + let reg = &mut data.nodes[nkey].value; + + if reg.activated { + return; + } + + reg.activated = true; + + data.activated.push_back(&mut data.nodes, nkey); + + if let Some(waker) = data.waker.take() { + waker.wake(); + } + } + + fn dispatch_activated(&self) { + // move the current list aside so we only process registrations that + // have been activated up to this point, and not registrations that + // might get activated during the course of calling callbacks + let mut activated = { + let data = &mut *self.data.borrow_mut(); + + let mut l = list::List::default(); + l.concat(&mut data.nodes, &mut data.activated); + + l + }; + + // call the callback of each activated registration, ensuring we + // release borrows before each call. this way, callbacks can access + // the eventloop, for example to add registrations + loop { + let (nkey, mut callback) = { + let data = &mut *self.data.borrow_mut(); + + let nkey = match activated.pop_front(&mut data.nodes) { + Some(nkey) => nkey, + None => break, + }; + + let reg = &mut data.nodes[nkey].value; + reg.activated = false; + + let callback = reg + .callback + .take() + .expect("registration should have a callback"); + + (nkey, callback) + }; + + callback.call(); + + let data = &mut *self.data.borrow_mut(); + + let reg = &mut data.nodes[nkey].value; + + reg.callback = Some(callback); + } + } + + fn set_waker(&self, waker: &Waker) { + let data = &mut *self.data.borrow_mut(); + + if let Some(current_waker) = &data.waker { + if !waker.will_wake(current_waker) { + // replace + data.waker = Some(waker.clone()); + } + } else { + // set + data.waker = Some(waker.clone()); + } + } +} + +struct Activator { + regs: Weak>, + reg_id: usize, +} + +impl waker::RcWake for Activator { + fn wake(self: Rc) { + if let Some(regs) = self.regs.upgrade() { + regs.activate(self.reg_id); + } + } +} + +#[derive(Debug)] +pub struct EventLoopError; + +pub struct EventLoop { + reactor: reactor::Reactor, + exit_code: Cell>, + regs: Rc>, +} + +impl EventLoop { + // will create a reactor if one does not exist in the current thread. if + // one already exists, registrations_max should be <= the max configured + // in the reactor. + pub fn new(registrations_max: usize) -> Self { + let reactor = if let Some(reactor) = reactor::Reactor::current() { + // use existing reactor if available + reactor + } else { + reactor::Reactor::new(registrations_max) + }; + + Self { + reactor, + exit_code: Cell::new(None), + regs: Rc::new(Registrations::new(registrations_max)), + } + } + + pub fn step(&self) -> Option { + self.poll_and_dispatch(Some(Duration::from_millis(0))) + } + + pub fn exec(&self) -> i32 { + loop { + if let Some(code) = self.poll_and_dispatch(None) { + break code; + } + } + } + + pub fn exec_async(&self) -> Exec { + Exec { l: self } + } + + pub fn exit(&self, code: i32) { + self.exit_code.set(Some(code)); + } + + pub fn register_fd( + &self, + fd: RawFd, + interest: u8, + callback: C, + ) -> Result { + let interest = if interest & READABLE != 0 && interest & WRITABLE != 0 { + mio::Interest::READABLE | mio::Interest::WRITABLE + } else if interest & READABLE != 0 { + mio::Interest::READABLE + } else if interest & WRITABLE != 0 { + mio::Interest::WRITABLE + } else { + // must specify at least one of READABLE or WRITABLE + return Err(EventLoopError); + }; + + let evented = match reactor::FdEvented::new(fd, interest, &self.reactor) { + Ok(evented) => evented, + Err(_) => return Err(EventLoopError), + }; + + let regs = Rc::downgrade(&self.regs); + + let get_waker = |reg_id| { + let activator = Rc::new(Activator { regs, reg_id }); + + waker::into_std(activator) + }; + + Ok(self + .regs + .add(Evented::Fd(evented), interest, get_waker, callback) + .expect("slab should have capacity")) + } + + pub fn register_timer(&self, timeout: Duration, callback: C) -> Result { + let expires = self.reactor.now() + timeout; + + let evented = match reactor::TimerEvented::new(expires, &self.reactor) { + Ok(evented) => evented, + Err(_) => return Err(EventLoopError), + }; + + let regs = Rc::downgrade(&self.regs); + + let get_waker = |reg_id| { + let activator = Rc::new(Activator { regs, reg_id }); + + waker::into_std(activator) + }; + + Ok(self + .regs + .add( + Evented::Timer(evented), + mio::Interest::READABLE, + get_waker, + callback, + ) + .expect("slab should have capacity")) + } + + pub fn deregister(&self, id: usize) { + self.regs.remove(id); + } + + fn poll_and_dispatch(&self, timeout: Option) -> Option { + // if exit code set, do a non-blocking poll + let timeout = if self.exit_code.get().is_some() { + Some(Duration::from_millis(0)) + } else { + timeout + }; + + self.reactor.poll(timeout).unwrap(); + self.regs.dispatch_activated(); + + self.exit_code.get() + } +} + +pub struct Exec<'a, C> { + l: &'a EventLoop, +} + +impl Future for Exec<'_, C> { + type Output = i32; + + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let l = self.l; + + l.regs.dispatch_activated(); + + if let Some(code) = l.exit_code.get() { + return Poll::Ready(code); + } + + l.regs.set_waker(cx.waker()); + + Poll::Pending + } +} + +mod ffi { + use super::*; + use std::ops::Deref; + + pub struct RawCallback { + // SAFETY: must be called with the associated ctx value + f: unsafe extern "C" fn(*mut libc::c_void), + + ctx: *mut libc::c_void, + } + + impl RawCallback { + // SAFETY: caller must ensure f is safe to call for the lifetime + // of the registration + pub unsafe fn new( + f: unsafe extern "C" fn(*mut libc::c_void), + ctx: *mut libc::c_void, + ) -> Self { + Self { f, ctx } + } + } + + impl Callback for RawCallback { + fn call(&mut self) { + // SAFETY: we are passing the ctx value that was provided + unsafe { + (self.f)(self.ctx); + } + } + } + + pub struct EventLoopRaw(EventLoop); + + impl Deref for EventLoopRaw { + type Target = EventLoop; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + #[no_mangle] + pub extern "C" fn event_loop_create(capacity: libc::c_uint) -> *mut EventLoopRaw { + let l = EventLoopRaw(EventLoop::new(capacity as usize)); + + Box::into_raw(Box::new(l)) + } + + #[allow(clippy::missing_safety_doc)] + #[no_mangle] + pub unsafe extern "C" fn event_loop_destroy(l: *mut EventLoopRaw) { + if !l.is_null() { + drop(Box::from_raw(l)); + } + } + + #[allow(clippy::missing_safety_doc)] + #[no_mangle] + pub unsafe extern "C" fn event_loop_exec(l: *mut EventLoopRaw) -> libc::c_int { + let l = l.as_mut().unwrap(); + + l.exec() as libc::c_int + } + + #[allow(clippy::missing_safety_doc)] + #[no_mangle] + pub unsafe extern "C" fn event_loop_exit(l: *mut EventLoopRaw, code: libc::c_int) { + let l = l.as_mut().unwrap(); + + l.exit(code); + } + + #[allow(clippy::missing_safety_doc)] + #[no_mangle] + pub unsafe extern "C" fn event_loop_register_fd( + l: *mut EventLoopRaw, + fd: std::os::raw::c_int, + interest: libc::c_uchar, + cb: unsafe extern "C" fn(*mut libc::c_void), + ctx: *mut libc::c_void, + out_id: *mut libc::size_t, + ) -> libc::c_int { + let l = l.as_mut().unwrap(); + + // SAFETY: we assume caller guarantees that the callback is safe to + // call for the lifetime of the registration + let cb = unsafe { RawCallback::new(cb, ctx) }; + + let id = match l.register_fd(fd, interest, cb) { + Ok(id) => id, + Err(_) => return -1, + }; + + out_id.write(id); + + 0 + } + + #[allow(clippy::missing_safety_doc)] + #[no_mangle] + pub unsafe extern "C" fn event_loop_register_timer( + l: *mut EventLoopRaw, + timeout: u64, + cb: unsafe extern "C" fn(*mut libc::c_void), + ctx: *mut libc::c_void, + out_id: *mut libc::size_t, + ) -> libc::c_int { + let l = l.as_mut().unwrap(); + + // SAFETY: we assume caller guarantees that the callback is safe to + // call for the lifetime of the registration + let cb = unsafe { RawCallback::new(cb, ctx) }; + + let id = match l.register_timer(Duration::from_millis(timeout), cb) { + Ok(id) => id, + Err(_) => return -1, + }; + + out_id.write(id); + + 0 + } + + #[allow(clippy::missing_safety_doc)] + #[no_mangle] + pub unsafe extern "C" fn event_loop_deregister(l: *mut EventLoopRaw, id: libc::size_t) { + let l = l.as_mut().unwrap(); + + l.deregister(id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::executor::Executor; + use crate::core::reactor::Reactor; + use std::os::fd::AsRawFd; + use std::rc::Rc; + + struct NoopCallback; + + impl Callback for NoopCallback { + fn call(&mut self) {} + } + + #[test] + fn exec() { + { + let l = EventLoop::::new(1); + assert_eq!(l.step(), None); + + l.exit(123); + assert_eq!(l.step(), Some(123)); + } + + { + let l = EventLoop::::new(1); + l.exit(124); + assert_eq!(l.exec(), 124); + } + } + + #[test] + fn fd() { + let l = Rc::new(EventLoop::>::new(1)); + + let listener = Rc::new(std::net::TcpListener::bind("127.0.0.1:0").unwrap()); + listener.set_nonblocking(true).unwrap(); + + let addr = listener.local_addr().unwrap(); + let fd = listener.as_raw_fd(); + + let cb = { + let l = Rc::clone(&l); + let listener = Rc::clone(&listener); + + Box::new(FnCallback(move || { + let _stream = listener.accept().unwrap(); + l.exit(0); + })) + }; + + let id = l.register_fd(fd, READABLE, cb).unwrap(); + + // non-blocking connect attempt to trigger listener + let _stream = mio::net::TcpStream::connect(addr); + + assert_eq!(l.exec(), 0); + + l.deregister(id); + } + + #[test] + fn timer() { + let l = Rc::new(EventLoop::>::new(1)); + + let cb = { + let l = Rc::clone(&l); + + Box::new(FnCallback(move || l.exit(0))) + }; + + let id = l.register_timer(Duration::from_millis(0), cb).unwrap(); + + // no space + assert!(l + .register_timer(Duration::from_millis(0), Box::new(NoopCallback)) + .is_err()); + + assert_eq!(l.exec(), 0); + + l.deregister(id); + + assert!(l + .register_timer(Duration::from_millis(0), Box::new(NoopCallback)) + .is_ok()); + } + + #[test] + fn exec_async() { + let reactor = Reactor::new(1); + let executor = Executor::new(1); + + executor + .spawn(async { + let l = Rc::new(EventLoop::>::new(1)); + + let listener = Rc::new(std::net::TcpListener::bind("127.0.0.1:0").unwrap()); + listener.set_nonblocking(true).unwrap(); + + let addr = listener.local_addr().unwrap(); + let fd = listener.as_raw_fd(); + + let cb = { + let l = Rc::clone(&l); + let listener = Rc::clone(&listener); + + Box::new(FnCallback(move || { + let _stream = listener.accept().unwrap(); + l.exit(0); + })) + }; + + let id = l.register_fd(fd, READABLE, cb).unwrap(); + + // non-blocking connect attempt to trigger listener + let _stream = mio::net::TcpStream::connect(addr); + + assert_eq!(l.exec_async().await, 0); + + l.deregister(id); + }) + .unwrap(); + + executor.run(|timeout| reactor.poll(timeout)).unwrap(); + } +} diff --git a/src/core/eventlooptest.cpp b/src/core/eventlooptest.cpp new file mode 100644 index 000000000..4eb2123c3 --- /dev/null +++ b/src/core/eventlooptest.cpp @@ -0,0 +1,114 @@ +/* +* Copyright (C) 2025 Fastly, Inc. +* +* This file is part of Pushpin. +* +* $FANOUT_BEGIN_LICENSE:APACHE2$ +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* $FANOUT_END_LICENSE$ +*/ + +#include +#include +#include +#include "defercall.h" +#include "eventloop.h" +#include "socketnotifier.h" +#include "timer.h" + +class EventLoopTest : public QObject +{ + Q_OBJECT + +private slots: + void cleanupTestCase() + { + DeferCall::cleanup(); + } + + void socketNotifier() + { + EventLoop loop(1); + + int fds[2]; + QCOMPARE(pipe(fds), 0); + + SocketNotifier *sn = new SocketNotifier(fds[0], SocketNotifier::Read); + + int activatedFd = -1; + sn->activated.connect([&](int fd) { + activatedFd = fd; + loop.exit(123); + }); + + unsigned char c = 1; + QCOMPARE(write(fds[1], &c, 1), 1); + + QCOMPARE(loop.exec(), 123); + QCOMPARE(activatedFd, fds[0]); + + delete sn; + close(fds[1]); + close(fds[0]); + } + + void timer() + { + EventLoop loop(2); + + Timer *t1 = new Timer; + Timer *t2 = new Timer; + + int timeoutCount = 0; + + t1->timeout.connect([&] { + ++timeoutCount; + }); + + t2->timeout.connect([&] { + ++timeoutCount; + loop.exit(123); + }); + + t1->setSingleShot(true); + t1->start(0); + + t2->setSingleShot(true); + t2->start(0); + + QCOMPARE(loop.exec(), 123); + QCOMPARE(timeoutCount, 2); + + delete t2; + delete t1; + } +}; + +namespace { +namespace Main { +QTEST_MAIN(EventLoopTest) +} +} + +extern "C" { + +int eventloop_test(int argc, char **argv) +{ + return Main::main(argc, argv); +} + +} + +#include "eventlooptest.moc" diff --git a/src/core/executor.rs b/src/core/executor.rs index d999eb21f..56bdcdb8b 100644 --- a/src/core/executor.rs +++ b/src/core/executor.rs @@ -195,10 +195,7 @@ impl Tasks { } fn take_task(&self, l: &mut list::List) -> Option<(usize, BoxFuture, Waker)> { - let nkey = match l.head { - Some(nkey) => nkey, - None => return None, - }; + let nkey = l.head?; let data = &mut *self.data.borrow_mut(); diff --git a/src/core/http1/client.rs b/src/core/http1/client.rs index ff1a56bbd..475f9b5fc 100644 --- a/src/core/http1/client.rs +++ b/src/core/http1/client.rs @@ -499,7 +499,7 @@ pub struct ResponseBody<'a, R: AsyncRead> { inner: RefCell>>, } -impl<'a, R: AsyncRead> ResponseBody<'a, R> { +impl ResponseBody<'_, R> { // on EOF and any subsequent calls, return success #[allow(clippy::await_holding_refcell_ref)] pub async fn add_to_buffer(&self) -> Result<(), Error> { @@ -647,7 +647,7 @@ pub struct FinishedKeepHeader<'a> { wbuf: &'a mut VecRingBuffer, } -impl<'a> FinishedKeepHeader<'a> { +impl FinishedKeepHeader<'_> { pub fn discard_header(self, resp: protocol::OwnedResponse) -> Finished { self.wbuf.set_inner(resp.into_buf()); self.wbuf.clear(); diff --git a/src/core/http1/protocol.rs b/src/core/http1/protocol.rs index e70f3c578..18f53ce0d 100644 --- a/src/core/http1/protocol.rs +++ b/src/core/http1/protocol.rs @@ -605,7 +605,7 @@ pub struct OwnedRequest<'s, const N: usize> { expect_100: bool, } -impl<'s, const N: usize> OwnedRequest<'s, N> { +impl OwnedRequest<'_, N> { pub fn get(&self) -> Request { let req = self.req.get(); @@ -640,7 +640,7 @@ pub struct OwnedResponse<'s, const N: usize> { body_size: BodySize, } -impl<'s, const N: usize> OwnedResponse<'s, N> { +impl OwnedResponse<'_, N> { pub fn get(&self) -> Response { let resp = self.resp.get(); diff --git a/src/core/http1/server.rs b/src/core/http1/server.rs index 2d149d631..2c2891c1b 100644 --- a/src/core/http1/server.rs +++ b/src/core/http1/server.rs @@ -469,7 +469,7 @@ pub struct ResponseState<'a, R: AsyncRead, W: AsyncWrite> { inner: RefCell>>, } -impl<'a, R: AsyncRead, W: AsyncWrite> Default for ResponseState<'a, R, W> { +impl Default for ResponseState<'_, R, W> { fn default() -> Self { Self { inner: RefCell::new(None), @@ -517,7 +517,7 @@ pub struct ResponsePrepareBody<'a, 'b, R: AsyncRead, W: AsyncWrite> { state: &'b RefCell>>, } -impl<'a, 'b, R: AsyncRead, W: AsyncWrite> ResponsePrepareBody<'a, 'b, R, W> { +impl ResponsePrepareBody<'_, '_, R, W> { // only returns an error on invalid input pub fn prepare(&mut self, src: &[u8], end: bool) -> Result<(usize, usize), Error> { let state = self.state.borrow(); @@ -623,7 +623,7 @@ pub struct ResponseBody<'a, R: AsyncRead, W: AsyncWrite> { inner: RefCell>>, } -impl<'a, R: AsyncRead, W: AsyncWrite> ResponseBody<'a, R, W> { +impl ResponseBody<'_, R, W> { pub fn prepare(&self, src: &[u8], end: bool) -> Result { if let Some(inner) = &*self.inner.borrow() { let w = &mut *inner.w.borrow_mut(); diff --git a/src/core/httprequest.h b/src/core/httprequest.h index 1901f8895..f41eecf2e 100644 --- a/src/core/httprequest.h +++ b/src/core/httprequest.h @@ -61,6 +61,7 @@ class HttpRequest : public QObject virtual void setIgnorePolicies(bool on) = 0; virtual void setTrustConnectHost(bool on) = 0; virtual void setIgnoreTlsErrors(bool on) = 0; + virtual void setTimeout(int msecs) = 0; virtual void start(const QString &method, const QUrl &uri, const HttpHeaders &headers) = 0; virtual void beginResponse(int code, const QByteArray &reason, const HttpHeaders &headers) = 0; diff --git a/src/core/io.rs b/src/core/io.rs index 073c82521..a74b9eba7 100644 --- a/src/core/io.rs +++ b/src/core/io.rs @@ -155,7 +155,7 @@ impl<'a, 'b, W: AsyncWrite> StdWriteWrapper<'a, 'b, W> { } } -impl<'a, 'b, W: AsyncWrite> Write for StdWriteWrapper<'a, 'b, W> { +impl Write for StdWriteWrapper<'_, '_, W> { fn write(&mut self, buf: &[u8]) -> Result { match self.w.as_mut().poll_write(self.cx, buf) { Poll::Ready(ret) => ret, @@ -244,7 +244,7 @@ pub struct ReadFuture<'a, R: AsyncRead + ?Sized> { buf: &'a mut [u8], } -impl<'a, R: AsyncRead + ?Sized> Future for ReadFuture<'a, R> { +impl Future for ReadFuture<'_, R> { type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { @@ -256,7 +256,7 @@ impl<'a, R: AsyncRead + ?Sized> Future for ReadFuture<'a, R> { } } -impl<'a, R: AsyncRead + ?Sized> Drop for ReadFuture<'a, R> { +impl Drop for ReadFuture<'_, R> { fn drop(&mut self) { self.r.cancel(); } @@ -268,7 +268,7 @@ pub struct WriteFuture<'a, W: AsyncWrite + ?Sized + Unpin> { pos: usize, } -impl<'a, W: AsyncWrite + ?Sized> Future for WriteFuture<'a, W> { +impl Future for WriteFuture<'_, W> { type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { @@ -291,7 +291,7 @@ impl<'a, W: AsyncWrite + ?Sized> Future for WriteFuture<'a, W> { } } -impl<'a, W: AsyncWrite + ?Sized> Drop for WriteFuture<'a, W> { +impl Drop for WriteFuture<'_, W> { fn drop(&mut self) { self.w.cancel(); } @@ -301,7 +301,7 @@ pub struct CloseFuture<'a, W: AsyncWrite + ?Sized> { w: &'a mut W, } -impl<'a, W: AsyncWrite + ?Sized> Future for CloseFuture<'a, W> { +impl Future for CloseFuture<'_, W> { type Output = Result<(), io::Error>; fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { @@ -313,7 +313,7 @@ impl<'a, W: AsyncWrite + ?Sized> Future for CloseFuture<'a, W> { } } -impl<'a, W: AsyncWrite + ?Sized> Drop for CloseFuture<'a, W> { +impl Drop for CloseFuture<'_, W> { fn drop(&mut self) { self.w.cancel(); } @@ -341,7 +341,7 @@ pub struct WriteVectoredFuture<'a, W: AsyncWrite + ?Sized + Unpin> { pos: usize, } -impl<'a, W: AsyncWrite + ?Sized> Future for WriteVectoredFuture<'a, W> { +impl Future for WriteVectoredFuture<'_, W> { type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { @@ -380,7 +380,7 @@ impl<'a, W: AsyncWrite + ?Sized> Future for WriteVectoredFuture<'a, W> { } } -impl<'a, W: AsyncWrite + ?Sized> Drop for WriteVectoredFuture<'a, W> { +impl Drop for WriteVectoredFuture<'_, W> { fn drop(&mut self) { self.w.cancel(); } @@ -391,7 +391,7 @@ pub struct WriteSharedFuture<'a, W: AsyncWrite + ?Sized + Unpin, B: AsRef<[u8]>> buf: &'a RefCell, } -impl<'a, W: AsyncWrite + ?Sized, B: AsRef<[u8]>> Future for WriteSharedFuture<'a, W, B> { +impl> Future for WriteSharedFuture<'_, W, B> { type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { @@ -403,7 +403,7 @@ impl<'a, W: AsyncWrite + ?Sized, B: AsRef<[u8]>> Future for WriteSharedFuture<'a } } -impl<'a, W: AsyncWrite + ?Sized, B: AsRef<[u8]>> Drop for WriteSharedFuture<'a, W, B> { +impl> Drop for WriteSharedFuture<'_, W, B> { fn drop(&mut self) { self.w.cancel(); } diff --git a/src/core/log.rs b/src/core/log.rs index b5ee8d89f..19e0758fc 100644 --- a/src/core/log.rs +++ b/src/core/log.rs @@ -18,9 +18,8 @@ use log::{Level, Log, Metadata, Record}; use std::fs::File; use std::io::{self, Write}; -use std::mem; use std::str; -use std::sync::{Mutex, Once}; +use std::sync::{Mutex, OnceLock}; use time::macros::format_description; use time::{OffsetDateTime, UtcOffset}; @@ -129,12 +128,10 @@ unsafe fn get_offset() -> Option { offset } -static mut LOGGER: mem::MaybeUninit = mem::MaybeUninit::uninit(); +static LOGGER: OnceLock = OnceLock::new(); pub fn ensure_init_simple_logger(output_file: Option, runner_mode: bool) { - static INIT: Once = Once::new(); - - INIT.call_once(|| { + LOGGER.get_or_init(|| { // SAFETY: we accept that this call is unsound. on some platforms it // is the only way to know the time zone, with a chance of UB if // another thread modifies environment vars during the call. the risk @@ -143,13 +140,10 @@ pub fn ensure_init_simple_logger(output_file: Option, runner_mode: bool) { // zone than not know the time zone let local_offset = unsafe { get_offset() }; - // SAFETY: call_once ensures this only happens from one place - unsafe { - LOGGER.write(SimpleLogger { - local_offset, - output_file: output_file.map(Mutex::new), - runner_mode, - }); + SimpleLogger { + local_offset, + output_file: output_file.map(Mutex::new), + runner_mode, } }); } @@ -157,8 +151,8 @@ pub fn ensure_init_simple_logger(output_file: Option, runner_mode: bool) { pub fn get_simple_logger() -> &'static SimpleLogger { ensure_init_simple_logger(None, false); - // SAFETY: logger is guaranteed to have been initialized - unsafe { LOGGER.assume_init_ref() } + // logger is guaranteed to have been initialized + LOGGER.get().expect("logger should be initialized") } pub fn local_offset_check() { diff --git a/src/core/mod.rs b/src/core/mod.rs index ee9aff8d1..7497dadca 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -20,6 +20,7 @@ pub mod channel; pub mod config; pub mod defer; pub mod event; +pub mod eventloop; pub mod executor; pub mod fs; pub mod http1; @@ -114,6 +115,11 @@ mod tests { unsafe { call_c_main(ffi::jwt_test, args) as u8 } } + fn eventloop_test(args: &[&OsStr]) -> u8 { + // SAFETY: safe to call + unsafe { call_c_main(ffi::eventloop_test, args) as u8 } + } + #[test] fn httpheaders() { assert!(qtest::run(httpheaders_test)); @@ -123,4 +129,9 @@ mod tests { fn jwt() { assert!(qtest::run(jwt_test)); } + + #[test] + fn eventloop() { + assert!(qtest::run(eventloop_test)); + } } diff --git a/src/core/packet/wscontrolpacket.cpp b/src/core/packet/wscontrolpacket.cpp index c87860053..0f89515f3 100644 --- a/src/core/packet/wscontrolpacket.cpp +++ b/src/core/packet/wscontrolpacket.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2022 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -206,6 +206,9 @@ QVariant WsControlPacket::toVariant() const if(!item.reason.isEmpty()) vitem["reason"] = item.reason; + if(item.debug) + vitem["debug"] = true; + if(!item.route.isEmpty()) vitem["route"] = item.route; @@ -218,6 +221,9 @@ QVariant WsControlPacket::toVariant() const if(item.logLevel >= 0) vitem["log-level"] = item.logLevel; + if(item.trusted) + vitem["trusted"] = true; + if(!item.channel.isEmpty()) vitem["channel"] = item.channel; @@ -359,6 +365,14 @@ bool WsControlPacket::fromVariant(const QVariant &in) item.reason = vitem["reason"].toByteArray(); } + if(vitem.contains("debug")) + { + if(typeId(vitem["debug"]) != QMetaType::Bool) + return false; + + item.debug = vitem["debug"].toBool(); + } + if(vitem.contains("route")) { if(typeId(vitem["route"]) != QMetaType::QByteArray) @@ -395,6 +409,14 @@ bool WsControlPacket::fromVariant(const QVariant &in) item.logLevel = vitem["log-level"].toInt(); } + if(vitem.contains("trusted")) + { + if(typeId(vitem["trusted"]) != QMetaType::Bool) + return false; + + item.trusted = vitem["trusted"].toBool(); + } + if(vitem.contains("channel")) { if(typeId(vitem["channel"]) != QMetaType::QByteArray) diff --git a/src/core/packet/wscontrolpacket.h b/src/core/packet/wscontrolpacket.h index ab8cf7f7c..29b1a6527 100644 --- a/src/core/packet/wscontrolpacket.h +++ b/src/core/packet/wscontrolpacket.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2022 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -61,10 +61,12 @@ class WsControlPacket bool queue; int code; QByteArray reason; + bool debug; QByteArray route; bool separateStats; QByteArray channelPrefix; int logLevel; + bool trusted; QByteArray channel; int ttl; int timeout; @@ -74,8 +76,10 @@ class WsControlPacket type((Type)-1), queue(false), code(-1), + debug(false), separateStats(false), logLevel(-1), + trusted(false), ttl(-1), timeout(-1) { diff --git a/src/core/processquit.cpp b/src/core/processquit.cpp index 43d848754..f2dcf12ee 100644 --- a/src/core/processquit.cpp +++ b/src/core/processquit.cpp @@ -38,57 +38,7 @@ # include #endif -namespace { - -// safeobj stuff, from qca - -void releaseAndDeleteLater(QObject *owner, QObject *obj) -{ - obj->disconnect(owner); - obj->setParent(0); - obj->deleteLater(); -} - -class SafeSocketNotifier : public QObject -{ - Q_OBJECT -public: - Connection activatedConnection; - - SafeSocketNotifier(int socket, QSocketNotifier::Type type, - QObject *parent = 0) : - QObject(parent) - { - sn = new QSocketNotifier(socket, type, this); - connect(sn, &QSocketNotifier::activated, this, &SafeSocketNotifier::doActivated); - } - - ~SafeSocketNotifier() - { - sn->setEnabled(false); - releaseAndDeleteLater(this, sn); - } - - bool isEnabled() const { return sn->isEnabled(); } - int socket() const { return sn->socket(); } - QSocketNotifier::Type type() const { return sn->type(); } - -public slots: - void setEnabled(bool enable) { sn->setEnabled(enable); } - -public: - SignalInt activated; - -private: - QSocketNotifier *sn; - - void doActivated(int sock) - { - activated(sock); - } -}; - -} +#include "socketnotifier.h" #ifndef NO_IRISNET namespace XMPP { @@ -120,7 +70,7 @@ class ProcessQuit::Private : public QObject #endif #ifdef Q_OS_UNIX int sig_pipe[2]; - SafeSocketNotifier *sig_notifier; + std::unique_ptr sig_notifier; #endif Private(ProcessQuit *_q) : QObject(_q), q(_q) @@ -138,7 +88,7 @@ class ProcessQuit::Private : public QObject return; } - sig_notifier = new SafeSocketNotifier(sig_pipe[0], QSocketNotifier::Read, this); + sig_notifier = std::make_unique(sig_pipe[0], SocketNotifier::Read); activatedConnection = sig_notifier->activated.connect(boost::bind(&Private::sig_activated, this, boost::placeholders::_1)); unixWatchAdd(SIGINT); unixWatchAdd(SIGHUP); @@ -157,7 +107,7 @@ class ProcessQuit::Private : public QObject unixWatchRemove(SIGHUP); unixWatchRemove(SIGTERM); activatedConnection.disconnect(); - delete sig_notifier; + sig_notifier.reset(); close(sig_pipe[0]); close(sig_pipe[1]); #endif diff --git a/src/core/qtest.rs b/src/core/qtest.rs index d3020763d..6b4f45a9f 100644 --- a/src/core/qtest.rs +++ b/src/core/qtest.rs @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,15 +55,18 @@ where let thread = if let Some(f) = output_file { let f = f.to_owned(); - let thread = thread::spawn(move || { - // this will block until the other side opens the file for writing - let f = File::open(&f).unwrap(); + let thread = thread::Builder::new() + .name("qtest-log".to_string()) + .spawn(move || { + // this will block until the other side opens the file for writing + let f = File::open(&f).unwrap(); - // forward the output until EOF or error - if let Err(e) = read_and_print_all(f) { - eprintln!("failed to read log line: {}", e); - } - }); + // forward the output until EOF or error + if let Err(e) = read_and_print_all(f) { + eprintln!("failed to read log line: {}", e); + } + }) + .unwrap(); Some(thread) } else { @@ -152,15 +155,18 @@ where let (s, r) = mpsc::channel::(); // run in the background forever - thread::spawn(move || { - for t in r { - let ret = call_qtest(t.f, output_file.as_deref()); - - // if receiver is gone, keep going - let _ = t.ret.send(ret); - } - unreachable!(); - }); + thread::Builder::new() + .name("qtest-run".to_string()) + .spawn(move || { + for t in r { + let ret = call_qtest(t.f, output_file.as_deref()); + + // if receiver is gone, keep going + let _ = t.ret.send(ret); + } + unreachable!(); + }) + .unwrap(); Mutex::new(s) }); diff --git a/src/core/qzmq/.gitignore b/src/core/qzmq/.gitignore deleted file mode 100644 index e91a03921..000000000 --- a/src/core/qzmq/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -conf.pri -Makefile -*.o -*.moc -moc_*.cpp -/examples/helloclient/helloclient -/examples/helloserver/helloserver diff --git a/src/core/qzmq/COPYING b/src/core/qzmq/COPYING deleted file mode 100644 index 15947bd14..000000000 --- a/src/core/qzmq/COPYING +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2012 Justin Karneges - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/src/core/qzmq/README b/src/core/qzmq/README deleted file mode 100644 index 0f75c175c..000000000 --- a/src/core/qzmq/README +++ /dev/null @@ -1,24 +0,0 @@ -QZmq ----- - -Author: Justin Karneges - -Yet another Qt binding for ZeroMQ. It wraps the C API of libzmq. It is -compatible with libzmq versions 2.x, 3.x, and 4.x. - -Some features: - - Completely event-driven, with both read and write notifications. - - For convenience, it is not necessary to create a Context explicitly. If a - Socket is created without one, then a globally shared Context will be - created automatically. - - Some handy extra classes. For example, RepRouter makes it easy to write a - REP socket server that handles multiple requests simultaneously, and Valve - makes it easy to regulate reads. - -To build the examples: - - echo "LIBS += -lzmq" > conf.pri - qmake && make - -To include the code in your project, just use the files in src. From a qmake -project you can include src.pri. It's your responsibility to link to libzmq. diff --git a/src/core/qzmq/examples/examples.pri b/src/core/qzmq/examples/examples.pri deleted file mode 100644 index c3b535f3b..000000000 --- a/src/core/qzmq/examples/examples.pri +++ /dev/null @@ -1,7 +0,0 @@ -exists($$PWD/../conf.pri):include($$PWD/../conf.pri) - -QT -= gui -QT += network - -INCLUDEPATH += $$PWD/../src -include($$PWD/../src/src.pri) diff --git a/src/core/qzmq/examples/examples.pro b/src/core/qzmq/examples/examples.pro deleted file mode 100644 index 3486346a7..000000000 --- a/src/core/qzmq/examples/examples.pro +++ /dev/null @@ -1,3 +0,0 @@ -TEMPLATE = subdirs - -SUBDIRS += helloclient helloserver diff --git a/src/core/qzmq/examples/helloclient/helloclient.cpp b/src/core/qzmq/examples/helloclient/helloclient.cpp deleted file mode 100644 index 10c0ab127..000000000 --- a/src/core/qzmq/examples/helloclient/helloclient.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include -#include -#include -#include "qzmqsocket.h" -#include - -using Connection = boost::signals2::scoped_connection; - -class App : public QObject -{ - Q_OBJECT - -private: - QZmq::Socket sock; - Connection rrConnection; - Connection mwConnection; - -public: - App() : - sock(QZmq::Socket::Req) - { - } - - void sock_messagesWritten(int count) - { - printf("messages written: %d\n", count); - } - - void sock_readyRead() - { - QList resp = sock.read(); - printf("read: %s\n", resp[0].data()); - emit quit(); - } - -public slots: - void start() - { - rrConnection = sock.readyRead.connect(boost::bind(&Private::sock_readyRead, this)); - mwConnection = sock.messagesWritten.connect(boost::bind(&Private::sock_messagesWritten, this, boost::placeholders::_1)); - sock.connectToAddress("tcp://localhost:5555"); - QByteArray out = "hello"; - printf("writing: %s\n", out.data()); - sock.write(QList() << out); - } - -signals: - void quit(); -}; - -int main(int argc, char **argv) -{ - QCoreApplication qapp(argc, argv); - App app; - QObject::connect(&app, SIGNAL(quit()), &qapp, SLOT(quit())); - QTimer::singleShot(0, &app, SLOT(start())); - return qapp.exec(); -} - -#include "helloclient.moc" diff --git a/src/core/qzmq/examples/helloclient/helloclient.pro b/src/core/qzmq/examples/helloclient/helloclient.pro deleted file mode 100644 index 80b8387d6..000000000 --- a/src/core/qzmq/examples/helloclient/helloclient.pro +++ /dev/null @@ -1,3 +0,0 @@ -include(../examples.pri) - -SOURCES += helloclient.cpp diff --git a/src/core/qzmq/examples/helloserver/helloserver.cpp b/src/core/qzmq/examples/helloserver/helloserver.cpp deleted file mode 100644 index b5bf59a06..000000000 --- a/src/core/qzmq/examples/helloserver/helloserver.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include -#include -#include -#include "qzmqreqmessage.h" -#include "qzmqreprouter.h" - -class App : public QObject -{ - Q_OBJECT - -private: - QZmq::RepRouter sock; - - void sock_messagesWritten(int count) - { - printf("messages written: %d\n", count); - } - - void sock_readyRead() - { - QZmq::ReqMessage msg = sock.read(); - if(msg.content().isEmpty()) - { - printf("error: received empty message\n"); - return; - } - - printf("read: %s\n", msg.content()[0].data()); - QByteArray out = "world"; - printf("writing: %s\n", out.data()); - sock.write(msg.createReply(QList() << out)); - } - -public slots: - void start() - { - rrConnection = sock.readyRead.connect(boost::bind(&Private::sock_readyRead, this)); - mwConnection = sock.messagesWritten.connect(boost::bind(&Private::sock_messagesWritten, this, boost::placeholders::_1)); - sock.bind("tcp://*:5555"); - } - -signals: - void quit(); -}; - -int main(int argc, char **argv) -{ - QCoreApplication qapp(argc, argv); - App app; - QObject::connect(&app, SIGNAL(quit()), &qapp, SLOT(quit())); - QTimer::singleShot(0, &app, SLOT(start())); - return qapp.exec(); -} - -#include "helloserver.moc" diff --git a/src/core/qzmq/examples/helloserver/helloserver.pro b/src/core/qzmq/examples/helloserver/helloserver.pro deleted file mode 100644 index 07fa1e902..000000000 --- a/src/core/qzmq/examples/helloserver/helloserver.pro +++ /dev/null @@ -1,3 +0,0 @@ -include(../examples.pri) - -SOURCES += helloserver.cpp diff --git a/src/core/qzmq/qzmq.pro b/src/core/qzmq/qzmq.pro deleted file mode 100644 index 922f87001..000000000 --- a/src/core/qzmq/qzmq.pro +++ /dev/null @@ -1,3 +0,0 @@ -TEMPLATE = subdirs - -SUBDIRS += examples diff --git a/src/core/qzmq/src/src.pri b/src/core/qzmq/src/src.pri deleted file mode 100644 index cbc8daf61..000000000 --- a/src/core/qzmq/src/src.pri +++ /dev/null @@ -1,12 +0,0 @@ -HEADERS += \ - $$PWD/qzmqcontext.h \ - $$PWD/qzmqsocket.h \ - $$PWD/qzmqvalve.h \ - $$PWD/qzmqreqmessage.h \ - $$PWD/qzmqreprouter.h - -SOURCES += \ - $$PWD/qzmqcontext.cpp \ - $$PWD/qzmqsocket.cpp \ - $$PWD/qzmqvalve.cpp \ - $$PWD/qzmqreprouter.cpp diff --git a/src/core/qzmq/src/qzmqcontext.cpp b/src/core/qzmqcontext.cpp similarity index 100% rename from src/core/qzmq/src/qzmqcontext.cpp rename to src/core/qzmqcontext.cpp diff --git a/src/core/qzmq/src/qzmqcontext.h b/src/core/qzmqcontext.h similarity index 100% rename from src/core/qzmq/src/qzmqcontext.h rename to src/core/qzmqcontext.h diff --git a/src/core/qzmq/src/qzmqreprouter.cpp b/src/core/qzmqreprouter.cpp similarity index 100% rename from src/core/qzmq/src/qzmqreprouter.cpp rename to src/core/qzmqreprouter.cpp diff --git a/src/core/qzmq/src/qzmqreprouter.h b/src/core/qzmqreprouter.h similarity index 94% rename from src/core/qzmq/src/qzmqreprouter.h rename to src/core/qzmqreprouter.h index 002509d3d..25e5c72f1 100644 --- a/src/core/qzmq/src/qzmqreprouter.h +++ b/src/core/qzmqreprouter.h @@ -24,9 +24,10 @@ #ifndef QZMQREPROUTER_H #define QZMQREPROUTER_H -#include #include +class QString; + using Signal = boost::signals2::signal; using SignalInt = boost::signals2::signal; using Connection = boost::signals2::scoped_connection; @@ -55,7 +56,8 @@ class RepRouter SignalInt messagesWritten; private: - Q_DISABLE_COPY(RepRouter) + RepRouter(const RepRouter &) = delete; + RepRouter &operator=(const RepRouter &) = delete; class Private; friend class Private; diff --git a/src/core/qzmq/src/qzmqreqmessage.h b/src/core/qzmqreqmessage.h similarity index 100% rename from src/core/qzmq/src/qzmqreqmessage.h rename to src/core/qzmqreqmessage.h diff --git a/src/core/qzmq/src/qzmqsocket.cpp b/src/core/qzmqsocket.cpp similarity index 94% rename from src/core/qzmq/src/qzmqsocket.cpp rename to src/core/qzmqsocket.cpp index d1e3f688c..447ae9343 100644 --- a/src/core/qzmq/src/qzmqsocket.cpp +++ b/src/core/qzmqsocket.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2012-2020 Justin Karneges - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the @@ -27,13 +27,12 @@ #include #include #include -#include -#include #include #include #include "rust/bindings.h" #include "qzmqcontext.h" -#include "rtimer.h" +#include "timer.h" +#include "socketnotifier.h" using Connection = boost::signals2::scoped_connection; @@ -362,27 +361,24 @@ static void removeGlobalContextRef() } } -class Socket::Private : public QObject +class Socket::Private { - Q_OBJECT - public: Socket *q; bool usingGlobalContext; Context *context; void *sock; - QSocketNotifier *sn_read; + std::unique_ptr sn_read; bool canWrite, canRead; QList< QList > pendingWrites; int pendingWritten; - std::unique_ptr updateTimer; + std::unique_ptr updateTimer; Connection updateTimerConnection; bool pendingUpdate; int shutdownWaitTime; bool writeQueueEnabled; Private(Socket *_q, Socket::Type type, Context *_context) : - QObject(_q), q(_q), canWrite(false), canRead(false), @@ -421,17 +417,19 @@ class Socket::Private : public QObject sock = wzmq_socket(context->context(), ztype); assert(sock != NULL); - sn_read = new QSocketNotifier(get_fd(sock), QSocketNotifier::Read, this); - connect(sn_read, &QSocketNotifier::activated, this, &Private::sn_read_activated); + sn_read = std::make_unique(get_fd(sock), SocketNotifier::Read); + sn_read->activated.connect(boost::bind(&Private::sn_read_activated, this)); sn_read->setEnabled(true); - updateTimer = std::make_unique(); + updateTimer = std::make_unique(); updateTimerConnection = updateTimer->timeout.connect(boost::bind(&Private::update_timeout, this)); updateTimer->setSingleShot(true); } ~Private() { + sn_read.reset(); + set_linger(sock, shutdownWaitTime); wzmq_close(sock); @@ -597,9 +595,9 @@ class Socket::Private : public QObject if(canRead) { - QPointer self = this; + std::weak_ptr self = q->d; q->readyRead(); - if(!self) + if(self.expired()) return; } @@ -619,7 +617,6 @@ class Socket::Private : public QObject doUpdate(); } -public slots: void sn_read_activated() { if(!processEvents()) @@ -635,22 +632,17 @@ public slots: } }; -Socket::Socket(Type type, QObject *parent) : - QObject(parent) +Socket::Socket(Type type) { - d = new Private(this, type, 0); + d = std::make_shared(this, type, nullptr); } -Socket::Socket(Type type, Context *context, QObject *parent) : - QObject(parent) +Socket::Socket(Type type, Context *context) { - d = new Private(this, type, context); + d = std::make_shared(this, type, context); } -Socket::~Socket() -{ - delete d; -} +Socket::~Socket() = default; void Socket::setShutdownWaitTime(int msecs) { @@ -772,5 +764,3 @@ void Socket::write(const QList &message) } } - -#include "qzmqsocket.moc" diff --git a/src/core/qzmq/src/qzmqsocket.h b/src/core/qzmqsocket.h similarity index 92% rename from src/core/qzmq/src/qzmqsocket.h rename to src/core/qzmqsocket.h index d4796efef..0c81a6a1f 100644 --- a/src/core/qzmq/src/qzmqsocket.h +++ b/src/core/qzmqsocket.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2012-2015 Justin Karneges - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the @@ -25,9 +25,12 @@ #ifndef QZMQSOCKET_H #define QZMQSOCKET_H -#include +#include +#include #include +class QString; + using Signal = boost::signals2::signal; using SignalInt = boost::signals2::signal; using Connection = boost::signals2::scoped_connection; @@ -36,10 +39,8 @@ namespace QZmq { class Context; -class Socket : public QObject +class Socket { - Q_OBJECT - public: enum Type { @@ -54,8 +55,8 @@ class Socket : public QObject Sub }; - Socket(Type type, QObject *parent = 0); - Socket(Type type, Context *context, QObject *parent = 0); + Socket(Type type); + Socket(Type type, Context *context); ~Socket(); // 0 means drop queue and don't block, -1 means infinite (default = -1) @@ -109,11 +110,12 @@ class Socket : public QObject SignalInt messagesWritten; private: - Q_DISABLE_COPY(Socket) + Socket(const Socket &) = delete; + Socket &operator=(const Socket &) = delete; class Private; friend class Private; - Private *d; + std::shared_ptr d; }; } diff --git a/src/core/qzmq/src/qzmqvalve.cpp b/src/core/qzmqvalve.cpp similarity index 86% rename from src/core/qzmq/src/qzmqvalve.cpp rename to src/core/qzmqvalve.cpp index e38497df8..f187fbc6f 100644 --- a/src/core/qzmq/src/qzmqvalve.cpp +++ b/src/core/qzmqvalve.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2012-2020 Justin Karneges + * Copyright (C) 2025 Fastly, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the @@ -23,15 +24,13 @@ #include "qzmqvalve.h" -#include #include "qzmqsocket.h" +#include "defercall.h" namespace QZmq { -class Valve::Private : public QObject +class Valve::Private { - Q_OBJECT - public: Valve *q; QZmq::Socket *sock; @@ -39,9 +38,9 @@ class Valve::Private : public QObject bool pendingRead; int maxReadsPerEvent; boost::signals2::scoped_connection rrConnection; + DeferCall deferCall; Private(Valve *_q) : - QObject(_q), q(_q), sock(0), isOpen(false), @@ -62,12 +61,12 @@ class Valve::Private : public QObject return; pendingRead = true; - QMetaObject::invokeMethod(this, "queuedRead", Qt::QueuedConnection); + deferCall.defer([=] { queuedRead(); }); } void tryRead() { - QPointer self = this; + std::weak_ptr self = q->d; int count = 0; while(isOpen && sock->canRead()) @@ -83,7 +82,7 @@ class Valve::Private : public QObject if(!msg.isEmpty()) { q->readyRead(msg); - if(!self) + if(self.expired()) return; } @@ -99,7 +98,6 @@ class Valve::Private : public QObject tryRead(); } -private slots: void queuedRead() { pendingRead = false; @@ -107,17 +105,13 @@ private slots: } }; -Valve::Valve(QZmq::Socket *sock, QObject *parent) : - QObject(parent) +Valve::Valve(QZmq::Socket *sock) { - d = new Private(this); + d = std::make_shared(this); d->setup(sock); } -Valve::~Valve() -{ - delete d; -} +Valve::~Valve() = default; bool Valve::isOpen() const { @@ -145,5 +139,3 @@ void Valve::close() } } - -#include "qzmqvalve.moc" diff --git a/src/core/qzmq/src/qzmqvalve.h b/src/core/qzmqvalve.h similarity index 91% rename from src/core/qzmq/src/qzmqvalve.h rename to src/core/qzmqvalve.h index ca63a5e60..4de805b76 100644 --- a/src/core/qzmq/src/qzmqvalve.h +++ b/src/core/qzmqvalve.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2012 Justin Karneges + * Copyright (C) 2025 Fastly, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the @@ -24,7 +25,8 @@ #ifndef QZMQVALVE_H #define QZMQVALVE_H -#include +#include +#include #include using SignalList = boost::signals2::signal&)>; @@ -34,12 +36,10 @@ namespace QZmq { class Socket; -class Valve : public QObject +class Valve { - Q_OBJECT - public: - Valve(QZmq::Socket *sock, QObject *parent = 0); + Valve(QZmq::Socket *sock); ~Valve(); bool isOpen() const; @@ -54,7 +54,7 @@ class Valve : public QObject private: class Private; friend class Private; - Private *d; + std::shared_ptr d; }; } diff --git a/src/core/reactor.rs b/src/core/reactor.rs index 1a0f23095..68cc3b4b2 100644 --- a/src/core/reactor.rs +++ b/src/core/reactor.rs @@ -39,7 +39,7 @@ fn duration_to_ticks_round_down(d: Duration) -> u64 { } fn duration_to_ticks_round_up(d: Duration) -> u64 { - ((d.as_millis() + (TICK_DURATION_MS as u128) - 1) / (TICK_DURATION_MS as u128)) as u64 + d.as_millis().div_ceil(TICK_DURATION_MS as u128) as u64 } fn ticks_to_duration(t: u64) -> Duration { diff --git a/src/core/redispool.cpp b/src/core/redispool.cpp new file mode 100644 index 000000000..2c91351b0 --- /dev/null +++ b/src/core/redispool.cpp @@ -0,0 +1,50 @@ +// RedisPool.cpp +#include "redispool.h" +#include "log.h" + +extern QString gRedisHostAddr; +extern int gRedisPort; +extern int gRedisPoolCount; + +RedisPool* RedisPool::instance() { + static RedisPool pool; + return &pool; +} + +RedisPool::RedisPool() {} + +RedisPool::~RedisPool() { + QMutexLocker locker(&m_mutex); + while (!m_pool.isEmpty()) { + redisFree(m_pool.dequeue()); + } +} + +redisContext* RedisPool::createConnection() { + return redisConnect(qPrintable(gRedisHostAddr), gRedisPort); // Update if needed +} + +QSharedPointer RedisPool::acquire() { + QMutexLocker locker(&m_mutex); + while (m_pool.isEmpty() && m_activeConnections >= gRedisPoolCount) { + m_cond.wait(&m_mutex); + } + + redisContext* conn = nullptr; + if (!m_pool.isEmpty()) { + conn = m_pool.dequeue(); + } else { + conn = createConnection(); + ++m_activeConnections; + } + + return QSharedPointer(conn, [](redisContext* c) { + RedisPool::instance()->release(c); + }); +} + +void RedisPool::release(redisContext* conn) { + QMutexLocker locker(&m_mutex); + m_pool.enqueue(conn); + m_cond.wakeOne(); +} diff --git a/src/core/redispool.h b/src/core/redispool.h new file mode 100644 index 000000000..227954a67 --- /dev/null +++ b/src/core/redispool.h @@ -0,0 +1,28 @@ +// RedisPool.h +#pragma once +#include +#include +#include +#include +#include +#include + +class RedisPool { +public: + static RedisPool* instance(); + + QSharedPointer acquire(); + void release(redisContext* conn); + +private: + RedisPool(); + ~RedisPool(); + + redisContext* createConnection(); + + QMutex m_mutex; + QWaitCondition m_cond; + QQueue m_pool; + const int m_maxConnections = 100; + int m_activeConnections = 0; +}; diff --git a/src/core/simplehttpserver.cpp b/src/core/simplehttpserver.cpp index bf395dbeb..ba5159752 100644 --- a/src/core/simplehttpserver.cpp +++ b/src/core/simplehttpserver.cpp @@ -30,6 +30,7 @@ #include #include #include "log.h" +#include "defercall.h" #include "httpheaders.h" class SimpleHttpRequest::Private : public QObject @@ -84,7 +85,7 @@ class SimpleHttpRequest::Private : public QObject { sock->disconnect(this); sock->setParent(0); - sock->deleteLater(); + DeferCall::deleteLater(sock); sock = 0; } } diff --git a/src/core/socketnotifier.cpp b/src/core/socketnotifier.cpp new file mode 100644 index 000000000..b1a7cbcc7 --- /dev/null +++ b/src/core/socketnotifier.cpp @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2025 Fastly, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "socketnotifier.h" + +#include "defercall.h" +#include "eventloop.h" + +SocketNotifier::SocketNotifier(int socket, Type type) : + socket_(socket), + type_(type), + enabled_(true), + inner_(nullptr), + loop_(EventLoop::instance()), + regId_(-1) +{ + if(loop_) + { + // if the rust-based eventloop is available, use it + + unsigned char interest = 0; + switch(type_) + { + case SocketNotifier::Read: + interest = EventLoop::Readable; + break; + case SocketNotifier::Write: + interest = EventLoop::Writable; + break; + } + + regId_ = loop_->registerFd(socket_, interest, SocketNotifier::cb_fd_activated, this); + } + else + { + // else fall back to qt eventloop + + QSocketNotifier::Type qType = type == Read ? QSocketNotifier::Read : QSocketNotifier::Write; + + inner_ = new QSocketNotifier(socket, qType); + connect(inner_, &QSocketNotifier::activated, this, &SocketNotifier::innerActivated); + } +} + +SocketNotifier::~SocketNotifier() +{ + if(inner_) + { + inner_->setEnabled(false); + + inner_->disconnect(this); + inner_->setParent(0); + DeferCall::deleteLater(inner_); + } + + if(regId_ >= 0) + loop_->deregister(regId_); +} + +void SocketNotifier::setEnabled(bool enable) +{ + enabled_ = enable; + + if(inner_) + inner_->setEnabled(enabled_); +} + +void SocketNotifier::innerActivated(int socket) +{ + activated(socket); +} + +void SocketNotifier::cb_fd_activated(void *ctx) +{ + SocketNotifier *self = (SocketNotifier *)ctx; + + self->fd_activated(); +} + +void SocketNotifier::fd_activated() +{ + if(enabled_) + activated(socket_); +} diff --git a/src/core/socketnotifier.h b/src/core/socketnotifier.h new file mode 100644 index 000000000..3df35e0ac --- /dev/null +++ b/src/core/socketnotifier.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2025 Fastly, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SOCKETNOTIFIER_H +#define SOCKETNOTIFIER_H + +#include +#include + +class EventLoop; + +class SocketNotifier : public QObject +{ + Q_OBJECT + +public: + enum Type + { + Read = 1, + Write = 2, + }; + + SocketNotifier(int socket, Type type); + ~SocketNotifier(); + + bool isEnabled() const { return enabled_; } + int socket() const { return socket_; } + Type type() const { return type_; } + + void setEnabled(bool enable); + + boost::signals2::signal activated; + +private slots: + void innerActivated(int socket); + +private: + int socket_; + Type type_; + bool enabled_; + QSocketNotifier *inner_; + EventLoop *loop_; + int regId_; + + static void cb_fd_activated(void *ctx); + void fd_activated(); +}; + +#endif diff --git a/src/core/statsmanager.cpp b/src/core/statsmanager.cpp index 504bde0b7..de0f8995a 100644 --- a/src/core/statsmanager.cpp +++ b/src/core/statsmanager.cpp @@ -26,17 +26,17 @@ #include #include #include -#include #include #include #include "qzmqsocket.h" #include "timerwheel.h" #include "log.h" +#include "defercall.h" #include "tnetstring.h" #include "httpheaders.h" #include "simplehttpserver.h" #include "zutil.h" -#include "rtimer.h" +#include "timer.h" // make this somewhat big since PUB is lossy #define OUT_HWM 200000 @@ -50,6 +50,25 @@ #define TICK_DURATION_MS 10 +extern QStringList gHttpBackendUrlList; +extern QStringList gWsBackendUrlList; + +extern quint32 numRequestReceived, numMessageSent, numWsConnect; +extern quint32 numClientCount, numHttpClientCount, numWsClientCount; +extern quint32 numRpcAuthor, numRpcBabe, numRpcBeefy, numRpcChain, numRpcChildState; +extern quint32 numRpcContracts, numRpcDev, numRpcEngine, numRpcEth, numRpcNet; +extern quint32 numRpcWeb3, numRpcGrandpa, numRpcMmr, numRpcOffchain, numRpcPayment; +extern quint32 numRpcRpc, numRpcState, numRpcSyncstate, numRpcSystem, numRpcSubscribe; +extern quint32 numCacheInsert, numCacheHit, numNeverTimeoutCacheInsert, numNeverTimeoutCacheHit; +extern quint32 numCacheLookup, numCacheExpiry, numRequestMultiPart; +extern quint32 numSubscriptionInsert, numSubscriptionHit, numSubscriptionLookup, numSubscriptionExpiry, numResponseMultiPart; +extern quint32 numCacheItem, numAutoRefreshItem, numAREItemCount, numSubscriptionItem, numNeverTimeoutCacheItem; +extern QHash groupMethodCountMap; +extern QHash httpCacheClientConnectFailedCountMap; +extern QHash httpCacheClientInvalidResponseCountMap; +extern QHash wsCacheClientConnectFailedCountMap; +extern QHash wsCacheClientInvalidResponseCountMap; + static qint64 durationToTicksRoundDown(qint64 msec) { return msec / TICK_DURATION_MS; @@ -365,11 +384,54 @@ class StatsManager::Private : public QObject public: enum Type { - RequestReceived, + RequestReceived, // 0 ConnectionConnected, ConnectionMinute, MessageReceived, - MessageSent + MessageSent, + numRequestReceived, + numMessageSent, + numWsConnect, + numClientCount, + numHttpClientCount, + numWsClientCount, // 10 + numRpcAuthor, + numRpcBabe, + numRpcBeefy, + numRpcChain, + numRpcChildState, + numRpcContracts, + numRpcDev, + numRpcEngine, + numRpcEth, + numRpcNet, // 20 + numRpcWeb3, + numRpcGrandpa, + numRpcMmr, + numRpcOffchain, + numRpcPayment, + numRpcRpc, + numRpcState, + numRpcSyncstate, + numRpcSystem, + numRpcSubscribe, // 30 + numCacheInsert, + numCacheHit, + numNeverTimeoutCacheInsert, + numNeverTimeoutCacheHit, + numCacheLookup, + numCacheExpiry, + numRequestMultiPart, + numSubscriptionInsert, + numSubscriptionHit, + numSubscriptionLookup, // 40 + numSubscriptionExpiry, + numResponseMultiPart, + numCacheItem, + numSubscriptionItem, + numNeverTimeoutCacheItem, + numAutoRefreshItem, + numAREItemCount // 47 }; Type mtype; @@ -403,7 +465,7 @@ class StatsManager::Private : public QObject int subscriptionTtl; int subscriptionLinger; int reportInterval; - QZmq::Socket *sock; + std::unique_ptr sock; SimpleHttpServer *prometheusServer; QString prometheusPrefix; QList prometheusMetrics; @@ -425,10 +487,10 @@ class StatsManager::Private : public QObject QHash reports; Counts combinedCounts; Report combinedReport; - std::unique_ptr activityTimer; - std::unique_ptr reportTimer; - std::unique_ptr refreshTimer; - std::unique_ptr externalConnectionsMaxTimer; + std::unique_ptr activityTimer; + std::unique_ptr reportTimer; + std::unique_ptr refreshTimer; + std::unique_ptr externalConnectionsMaxTimer; Connection activityTimerConnection; Connection reportTimerConnection; Connection refreshTimerConnection; @@ -450,21 +512,20 @@ class StatsManager::Private : public QObject subscriptionTtl(60 * 1000), subscriptionLinger(60 * 1000), reportInterval(10 * 1000), - sock(0), prometheusServer(0), currentConnectionInfoRefreshBucket(0), currentSubscriptionRefreshBucket(0), wheel(TimerWheel((_connectionsMax * 2) + _subscriptionsMax)) { - activityTimer = std::make_unique(); + activityTimer = std::make_unique(); activityTimerConnection = activityTimer->timeout.connect(boost::bind(&Private::activity_timeout, this)); activityTimer->setSingleShot(true); - refreshTimer = std::make_unique(); + refreshTimer = std::make_unique(); refreshTimerConnection = refreshTimer->timeout.connect(boost::bind(&Private::refresh_timeout, this)); refreshTimer->start(REFRESH_INTERVAL); - externalConnectionsMaxTimer = std::make_unique(); + externalConnectionsMaxTimer = std::make_unique(); externalConnectionsMaxTimerConnection = externalConnectionsMaxTimer->timeout.connect(boost::bind(&Private::externalConnectionsMax_timeout, this)); externalConnectionsMaxTimer->start(EXTERNAL_CONNECTIONS_MAX_INTERVAL); @@ -476,6 +537,77 @@ class StatsManager::Private : public QObject prometheusMetrics += PrometheusMetric(PrometheusMetric::ConnectionMinute, "connection_minute", "counter", "Number of minutes clients have been connected"); prometheusMetrics += PrometheusMetric(PrometheusMetric::MessageReceived, "message_received", "counter", "Number of messages received by the publish API"); prometheusMetrics += PrometheusMetric(PrometheusMetric::MessageSent,"message_sent", "counter", "Number of messages sent to clients"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRequestReceived, "number_of_ws_request_received", "counter", "Number of ws requests received"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numMessageSent, "number_of_ws_message_sent", "counter", "Number of ws message sent"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numWsConnect, "number_of_ws_connection_received", "counter", "Number of ws sconcurrent connections"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numClientCount, "number_of_client_count", "counter", "Number of connecting clients"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numHttpClientCount, "number_of_client_count_http", "counter", "Number of http connecting clients"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numWsClientCount, "number_of_client_count_ws", "counter", "Number of ws connecting clients"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcAuthor, "number_of_group_author", "counter", "Number of ws JSON-RPC author method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcBabe, "number_of_group_babe", "counter", "Number of ws JSON-RPC babe method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcBeefy, "number_of_group_beefy", "counter", "Number of ws JSON-RPC beefy method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcChain, "number_of_group_chain", "counter", "Number of ws JSON-RPC chain method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcChildState, "number_of_group_childstate", "counter", "Number of ws JSON-RPC childstate method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcContracts, "number_of_group_contracts", "counter", "Number of ws JSON-RPC contracts method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcDev, "number_of_group_dev", "counter", "Number of ws JSON-RPC dev method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcEngine, "number_of_group_engine", "counter", "Number of ws JSON-RPC engine method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcEth, "number_of_group_eth", "counter", "Number of ws JSON-RPC eth method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcNet, "number_of_group_net_eth", "counter", "Number of ws JSON-RPC net_eth method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcWeb3, "number_of_group_web3_eth", "counter", "Number of ws JSON-RPC web3_eth method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcGrandpa, "number_of_group_grandpa", "counter", "Number of ws JSON-RPC grandpa method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcMmr, "number_of_group_mmr", "counter", "Number of ws JSON-RPC mmr method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcOffchain, "number_of_group_offchain", "counter", "Number of ws JSON-RPC offchain method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcPayment, "number_of_group_paymenet", "counter", "Number of ws JSON-RPC payment method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcRpc, "number_of_group_rpc", "counter", "Number of ws JSON-RPC rpc method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcState, "number_of_group_state", "counter", "Number of ws JSON-RPC state method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcSyncstate, "number_of_group_syncstate", "counter", "Number of ws JSON-RPC syncstate method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcSystem, "number_of_group_system", "counter", "Number of ws JSON-RPC system method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRpcSubscribe, "number_of_group_subscribe", "counter", "Number of ws JSON-RPC subscribe method group"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numCacheInsert, "number_of_cache_insert", "counter", "Number of ws Cache insert event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numCacheHit, "number_of_cache_hit", "counter", "Number of ws Cache hit event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numNeverTimeoutCacheInsert, "number_of_never_timeout_cache_insert", "counter", "Number of ws Never Timeout Cache insert event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numNeverTimeoutCacheHit, "number_of_never_timeout_cache_hit", "counter", "Number of ws Never Timeout Cache hit event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numCacheLookup, "number_of_cache_lookup", "counter", "Number of ws Cache lookup event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numCacheExpiry, "number_of_cache_expiry", "counter", "Number of ws Cache expiry event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numRequestMultiPart, "number_of_cache_request_multi_part", "counter", "Number of ws Cache multi-part request"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numSubscriptionInsert, "number_of_subscription_insert", "counter", "Number of ws Subscripion insert event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numSubscriptionHit, "number_of_subscription_hit", "counter", "Number of ws Subscripion hit event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numSubscriptionLookup, "number_of_subscription_lookup", "counter", "Number of ws Subscripion lookup event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numSubscriptionExpiry, "number_of_subscription_expiry", "counter", "Number of ws Subscripion expiry event"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numResponseMultiPart, "number_of_cache_response_multi_part", "counter", "Number of ws Subscripion multi-part response"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numCacheItem, "number_of_cache_item", "counter", "Number of ws Cache items"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numSubscriptionItem, "number_of_subscription_item", "counter", "Number of ws Subscription items"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numNeverTimeoutCacheItem, "number_of_never_timeout_cache_item", "counter", "Number of ws Never Timeout Cache items"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numAutoRefreshItem, "number_of_cache_auto_refresh_item", "counter", "Number of ws Auto-Refresh items"); + prometheusMetrics += PrometheusMetric(PrometheusMetric::numAREItemCount, "number_of_cache_auto_refresh_exception_item", "counter", "Number of ws Auto-Refresh Exception items"); + + // user-defined method group count + int mapCnt = 48; + foreach(QString groupKey, groupMethodCountMap.keys()) + { + prometheusMetrics += PrometheusMetric((PrometheusMetric::Type)(mapCnt), "number_of_" + groupKey, "counter", "Number of ws "+groupKey); + mapCnt++; + } + for (int i=0; i(QZmq::Socket::Pub); sock->setHwm(OUT_HWM); sock->setWriteQueueEnabled(false); sock->setShutdownWaitTime(0); QString errorMessage; - if(!ZUtil::setupSocket(sock, spec, true, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(sock.get(), spec, true, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; @@ -607,7 +739,7 @@ class StatsManager::Private : public QObject { if(reportInterval > 0 && !reportTimer) { - reportTimer = std::make_unique(); + reportTimer = std::make_unique(); reportTimerConnection = reportTimer->timeout.connect(boost::bind(&Private::report_timeout, this)); reportTimer->start(reportInterval); } @@ -1548,6 +1680,86 @@ class StatsManager::Private : public QObject case PrometheusMetric::ConnectionMinute: value = QVariant(combinedReport.connectionsMinutes); break; case PrometheusMetric::MessageReceived: value = QVariant(combinedReport.messagesReceived); break; case PrometheusMetric::MessageSent: value = QVariant(combinedReport.messagesSent); break; + case PrometheusMetric::numRequestReceived: value = QVariant(numRequestReceived); break; + case PrometheusMetric::numMessageSent: value = QVariant(numMessageSent); break; + case PrometheusMetric::numWsConnect: value = QVariant(numWsConnect); break; + case PrometheusMetric::numClientCount: value = QVariant(numClientCount); break; + case PrometheusMetric::numHttpClientCount: value = QVariant(numHttpClientCount); break; + case PrometheusMetric::numWsClientCount: value = QVariant(numWsClientCount); break; + case PrometheusMetric::numRpcAuthor: value = QVariant(numRpcAuthor); break; + case PrometheusMetric::numRpcBabe: value = QVariant(numRpcBabe); break; + case PrometheusMetric::numRpcBeefy: value = QVariant(numRpcBeefy); break; + case PrometheusMetric::numRpcChain: value = QVariant(numRpcChain); break; + case PrometheusMetric::numRpcChildState: value = QVariant(numRpcChildState); break; + case PrometheusMetric::numRpcContracts: value = QVariant(numRpcContracts); break; + case PrometheusMetric::numRpcDev: value = QVariant(numRpcDev); break; + case PrometheusMetric::numRpcEngine: value = QVariant(numRpcEngine); break; + case PrometheusMetric::numRpcEth: value = QVariant(numRpcEth); break; + case PrometheusMetric::numRpcNet: value = QVariant(numRpcNet); break; + case PrometheusMetric::numRpcWeb3: value = QVariant(numRpcWeb3); break; + case PrometheusMetric::numRpcGrandpa: value = QVariant(numRpcGrandpa); break; + case PrometheusMetric::numRpcMmr: value = QVariant(numRpcMmr); break; + case PrometheusMetric::numRpcOffchain: value = QVariant(numRpcOffchain); break; + case PrometheusMetric::numRpcPayment: value = QVariant(numRpcPayment); break; + case PrometheusMetric::numRpcRpc: value = QVariant(numRpcRpc); break; + case PrometheusMetric::numRpcState: value = QVariant(numRpcState); break; + case PrometheusMetric::numRpcSyncstate: value = QVariant(numRpcSyncstate); break; + case PrometheusMetric::numRpcSystem: value = QVariant(numRpcSystem); break; + case PrometheusMetric::numRpcSubscribe: value = QVariant(numRpcSubscribe); break; + case PrometheusMetric::numCacheInsert: value = QVariant(numCacheInsert); break; + case PrometheusMetric::numCacheHit: value = QVariant(numCacheHit); break; + case PrometheusMetric::numNeverTimeoutCacheInsert: value = QVariant(numNeverTimeoutCacheInsert); break; + case PrometheusMetric::numNeverTimeoutCacheHit: value = QVariant(numNeverTimeoutCacheHit); break; + case PrometheusMetric::numCacheLookup: value = QVariant(numCacheLookup); break; + case PrometheusMetric::numCacheExpiry: value = QVariant(numCacheExpiry); break; + case PrometheusMetric::numRequestMultiPart: value = QVariant(numRequestMultiPart); break; + case PrometheusMetric::numSubscriptionInsert: value = QVariant(numSubscriptionInsert); break; + case PrometheusMetric::numSubscriptionHit: value = QVariant(numSubscriptionHit); break; + case PrometheusMetric::numSubscriptionLookup: value = QVariant(numSubscriptionLookup); break; + case PrometheusMetric::numSubscriptionExpiry: value = QVariant(numSubscriptionExpiry); break; + case PrometheusMetric::numResponseMultiPart: value = QVariant(numResponseMultiPart); break; + case PrometheusMetric::numCacheItem: value = QVariant(numCacheItem); break; + case PrometheusMetric::numSubscriptionItem: value = QVariant(numSubscriptionItem); break; + case PrometheusMetric::numNeverTimeoutCacheItem: value = QVariant(numNeverTimeoutCacheItem); break; + case PrometheusMetric::numAutoRefreshItem: value = QVariant(numAutoRefreshItem); break; + case PrometheusMetric::numAREItemCount: value = QVariant(numAREItemCount); break; + default: + int currCnt = 48; + if (m.mtype >= currCnt && m.mtype < (currCnt+groupMethodCountMap.size())) + { + int typeNum = m.mtype - currCnt; + value = QVariant(groupMethodCountMap.values()[typeNum]); + } + currCnt += groupMethodCountMap.size(); + if (m.mtype >= currCnt && m.mtype < (currCnt+httpCacheClientConnectFailedCountMap.size())) + { + int typeNum = m.mtype - currCnt; + QString key = gHttpBackendUrlList[typeNum]; + value = QVariant(httpCacheClientConnectFailedCountMap[key]); + } + currCnt += httpCacheClientConnectFailedCountMap.size(); + if (m.mtype >= currCnt && m.mtype < (currCnt+httpCacheClientInvalidResponseCountMap.size())) + { + int typeNum = m.mtype - currCnt; + QString key = gHttpBackendUrlList[typeNum]; + value = QVariant(httpCacheClientInvalidResponseCountMap[key]); + } + currCnt += httpCacheClientInvalidResponseCountMap.size(); + if (m.mtype >= currCnt && m.mtype < (currCnt+wsCacheClientConnectFailedCountMap.size())) + { + int typeNum = m.mtype - currCnt; + QString key = gWsBackendUrlList[typeNum]; + value = QVariant(wsCacheClientConnectFailedCountMap[key]); + } + currCnt += wsCacheClientConnectFailedCountMap.size(); + if (m.mtype >= currCnt && m.mtype < (currCnt+wsCacheClientInvalidResponseCountMap.size())) + { + int typeNum = m.mtype - currCnt; + QString key = gWsBackendUrlList[typeNum]; + value = QVariant(wsCacheClientInvalidResponseCountMap[key]); + } + currCnt += wsCacheClientInvalidResponseCountMap.size(); + break; } if(value.isNull()) @@ -1560,7 +1772,7 @@ class StatsManager::Private : public QObject ).arg(prometheusPrefix, m.name, m.help, prometheusPrefix, m.name, m.type, prometheusPrefix, m.name, value.toString()); } - req->finished.connect(boost::bind(&SimpleHttpRequest::deleteLater, req)); + req->finished.connect([=] { DeferCall::deleteLater(req); }); HttpHeaders headers; headers += HttpHeader("Content-Type", "text/plain"); diff --git a/src/core/tests.pri b/src/core/tests.pri index 391300291..1fe8d960a 100644 --- a/src/core/tests.pri +++ b/src/core/tests.pri @@ -3,4 +3,5 @@ INCLUDES += \ SOURCES += \ $$PWD/httpheaderstest.cpp \ - $$PWD/jwttest.cpp + $$PWD/jwttest.cpp \ + $$PWD/eventlooptest.cpp diff --git a/src/core/rtimer.cpp b/src/core/timer.cpp similarity index 78% rename from src/core/rtimer.cpp rename to src/core/timer.cpp index 5dcbe292d..6bc93f73d 100644 --- a/src/core/rtimer.cpp +++ b/src/core/timer.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2021 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -21,12 +21,13 @@ * $FANOUT_END_LICENSE$ */ -#include "rtimer.h" +#include "timer.h" #include #include #include #include "timerwheel.h" +#include "eventloop.h" #define TICK_DURATION_MS 10 #define UPDATE_TICKS_MAX 1000 @@ -54,7 +55,7 @@ class TimerManager : public QObject public: TimerManager(int capacity, QObject *parent = 0); - int add(int msec, RTimer *r); + int add(int msec, Timer *r); void remove(int key); private slots: @@ -81,7 +82,7 @@ TimerManager::TimerManager(int capacity, QObject *parent) : t_->setSingleShot(true); } -int TimerManager::add(int msec, RTimer *r) +int TimerManager::add(int msec, Timer *r) { qint64 currentTime = QDateTime::currentMSecsSinceEpoch(); @@ -130,7 +131,7 @@ void TimerManager::t_timeout() break; } - RTimer *r = (RTimer *)expired.userData; + Timer *r = (Timer *)expired.userData; r->timerReady(); } @@ -173,65 +174,94 @@ void TimerManager::updateTimeout(qint64 currentTime) static thread_local TimerManager *g_manager = 0; -RTimer::RTimer() : +Timer::Timer() : + loop_(EventLoop::instance()), singleShot_(false), interval_(0), timerId_(-1) { } -RTimer::~RTimer() +Timer::~Timer() { stop(); } -bool RTimer::isActive() const +bool Timer::isActive() const { return (timerId_ >= 0); } -void RTimer::setSingleShot(bool singleShot) +void Timer::setSingleShot(bool singleShot) { singleShot_ = singleShot; } -void RTimer::setInterval(int msec) +void Timer::setInterval(int msec) { interval_ = msec; } -void RTimer::start(int msec) +void Timer::start(int msec) { setInterval(msec); start(); } -void RTimer::start() +void Timer::start() { - // must call RTimer::init first - assert(g_manager); - stop(); - int id = g_manager->add(interval_, this); - assert(id >= 0); + if(loop_) + { + // if the rust-based eventloop is available, use it + + int id = loop_->registerTimer(interval_, Timer::cb_timer_activated, this); + assert(id >= 0); + + timerId_ = id; + } + else + { + // else fall back to qt eventloop + + // must call Timer::init first + assert(g_manager); + + int id = g_manager->add(interval_, this); + assert(id >= 0); - timerId_ = id; + timerId_ = id; + } } -void RTimer::stop() +void Timer::stop() { if(timerId_ >= 0) { - assert(g_manager); + if(loop_) + { + loop_->deregister(timerId_); + } + else + { + assert(g_manager); - g_manager->remove(timerId_); + g_manager->remove(timerId_); + } timerId_ = -1; } } -void RTimer::timerReady() +void Timer::cb_timer_activated(void *ctx) +{ + Timer *self = (Timer *)ctx; + + self->timerReady(); +} + +void Timer::timerReady() { timerId_ = -1; @@ -243,17 +273,17 @@ void RTimer::timerReady() timeout(); } -void RTimer::init(int capacity) +void Timer::init(int capacity) { assert(!g_manager); g_manager = new TimerManager(capacity); } -void RTimer::deinit() +void Timer::deinit() { delete g_manager; g_manager = 0; } -#include "rtimer.moc" +#include "timer.moc" diff --git a/src/core/rtimer.h b/src/core/timer.h similarity index 83% rename from src/core/rtimer.h rename to src/core/timer.h index 2032a6b3f..f197c11ca 100644 --- a/src/core/rtimer.h +++ b/src/core/timer.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2021 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -21,23 +21,24 @@ * $FANOUT_END_LICENSE$ */ -#ifndef RTIMER_H -#define RTIMER_H +#ifndef TIMER_H +#define TIMER_H #include #include using Signal = boost::signals2::signal; +class EventLoop; class TimerManager; -class RTimer : public QObject +class Timer : public QObject { Q_OBJECT public: - RTimer(); - ~RTimer(); + Timer(); + ~Timer(); bool isActive() const; @@ -50,7 +51,7 @@ class RTimer : public QObject // initialization is thread local static void init(int capacity); - // only call if there are no active RTimers + // only call if there are no active timers static void deinit(); Signal timeout; @@ -58,10 +59,12 @@ class RTimer : public QObject private: friend class TimerManager; + EventLoop *loop_; bool singleShot_; int interval_; int timerId_; + static void cb_timer_activated(void *ctx); void timerReady(); }; diff --git a/src/core/wscatworker.cpp b/src/core/wscatworker.cpp new file mode 100644 index 000000000..0f9d754c7 --- /dev/null +++ b/src/core/wscatworker.cpp @@ -0,0 +1,47 @@ +// wscatworker.cpp +#include "wscatworker.h" +#include +#include + +#include "log.h" + +WscatWorker::WscatWorker(QObject *parent) : QObject(parent), process(nullptr) {} + +WscatWorker::~WscatWorker() { + stopWscat(); +} + +void WscatWorker::startWscat(const QString &url, const QStringList &headers) { + + QThread::msleep(100); + QMutexLocker locker(&mutex); + if (process) return; + + process = new QProcess(this); + + QStringList args; + for (const QString& h : headers) { + args << "--header" << h; + } + args << "-c" << url; + + process->start("wscat", args); + if (!process->waitForStarted()) { + log_debug("[WS] Failed to start wscat"); + delete process; + process = nullptr; + } + + process->waitForFinished(-1); // Wait until wscat exits +} + +void WscatWorker::stopWscat() { + QMutexLocker locker(&mutex); + if (process) { + log_debug("[WS] killing wscat process"); + process->kill(); // or process->terminate() for graceful + process->waitForFinished(3000); + process->deleteLater(); + process = nullptr; + } +} diff --git a/src/core/wscatworker.h b/src/core/wscatworker.h new file mode 100644 index 000000000..42faf4fc6 --- /dev/null +++ b/src/core/wscatworker.h @@ -0,0 +1,24 @@ +// wscatworker.h +#pragma once + +#include +#include +#include + +class WscatWorker : public QObject { + Q_OBJECT +public: + explicit WscatWorker(QObject *parent = nullptr); + ~WscatWorker(); + +public slots: + void startWscat(const QString &url, const QStringList &args); + void stopWscat(); + +signals: + void finished(); + +private: + QProcess *process; + QMutex mutex; +}; diff --git a/src/core/zhttpmanager.cpp b/src/core/zhttpmanager.cpp index d3ccaa3bd..c3a005f01 100644 --- a/src/core/zhttpmanager.cpp +++ b/src/core/zhttpmanager.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2012-2021 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -25,7 +26,15 @@ #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "qzmqsocket.h" #include "qzmqvalve.h" #include "tnetstring.h" @@ -34,7 +43,8 @@ #include "log.h" #include "zutil.h" #include "logutil.h" -#include "rtimer.h" +#include "timer.h" +#include "cacheutil.h" #define OUT_HWM 100 #define IN_HWM 100 @@ -54,6 +64,107 @@ // needs to match the peer #define ZHTTP_IDS_MAX 128 +// max length of one packet in log +#define DEBUG_LOG_MAX_LENGTH 1024 + +#define CACHE_INTERVAL 1000 + +#define PING_INTERVAL 20 + +///////////////////////////////////////////////////////////////////////////////////// +// cache data structure + +bool gCacheEnable = false; +QStringList gHttpBackendUrlList; +QStringList gWsBackendUrlList; + +QList gWsCacheClientList; +QHash gWsKilledCacheClientMap; +QHash gRestartCacheClientMap; + +ZhttpResponsePacket gWsInitResponsePacket; +QHash gWsClientMap; +QHash gHttpClientMap; + +QList gCacheKeyItemList; +QString gMsgIdAttrName = "id"; +QString gMsgMethodAttrName = "method"; +QString gMsgParamsAttrName = "params"; +QString gResultAttrName = "result"; +QStringList gErrorAttrList; +QString gSubscriptionAttrName = "params>>subscription"; +QString gSubscribeBlockAttrName = "params>>result>>block"; +QString gSubscribeChangesAttrName = "params>>result>>changes"; + +int gAccessTimeoutSeconds = 30; +int gResponseTimeoutSeconds = 90; +int gClientNoRequestTimeoutSeconds = 120; // 2mins +int gCacheTimeoutSeconds = 20; +int gShorterTimeoutSeconds = 10; +int gLongerTimeoutSeconds = 60; +int gCacheItemMaxCount = 3000; + +QFuture gCacheThread; + +QStringList gCacheMethodList; +QHash gSubscribeMethodMap; +QHash> gUnsubscribeRequestMap; +QList gDeleteClientList; +QStringList gNeverTimeoutMethodList; +QStringList gRefreshShorterMethodList; +QStringList gRefreshLongerMethodList; +QStringList gRefreshUneraseMethodList; +QStringList gRefreshExcludeMethodList; +QStringList gRefreshPassthroughMethodList; +QStringList gNullResponseMethodList; + +// multi packets params +QHash gHttpMultiPartResponseItemMap; +QHash gWsMultiPartRequestItemMap; +QHash gWsMultiPartResponseItemMap; + +// time seconds to retry another backend for null response +int gBackendSwitchIntervalSeconds = 10; + +// prometheus restore allow seconds (default 300) +int gPrometheusRestoreAllowSeconds = 300; + +// redis +bool gRedisEnable = false; +QString gRedisHostAddr = "127.0.0.1"; +int gRedisPort = 6379; +int gRedisPoolCount = 10; +QString gRedisKeyHeader = ""; +bool gReplicaFlag = false; +QString gReplicaMasterAddr = ""; +int gReplicaMasterPort = 6379; +bool gReplicaTimerStarted = false; + +// count method group +QHash gCountMethodGroupMap; + +// prometheus status +QList gCacheMethodResponseCountList; +quint32 numRequestReceived, numMessageSent, numWsConnect; +quint32 numClientCount, numHttpClientCount, numWsClientCount; +quint32 numRpcAuthor, numRpcBabe, numRpcBeefy, numRpcChain, numRpcChildState; +quint32 numRpcContracts, numRpcDev, numRpcEngine, numRpcEth, numRpcNet; +quint32 numRpcWeb3, numRpcGrandpa, numRpcMmr, numRpcOffchain, numRpcPayment; +quint32 numRpcRpc, numRpcState, numRpcSyncstate, numRpcSystem, numRpcSubscribe; +quint32 numCacheInsert, numCacheHit, numNeverTimeoutCacheInsert, numNeverTimeoutCacheHit; +quint32 numCacheLookup, numCacheExpiry, numRequestMultiPart; +quint32 numSubscriptionInsert, numSubscriptionHit, numSubscriptionLookup, numSubscriptionExpiry, numResponseMultiPart; +quint32 numCacheItem, numAutoRefreshItem, numAREItemCount, numSubscriptionItem, numNeverTimeoutCacheItem; +QHash groupMethodCountMap; +QHash httpCacheClientConnectFailedCountMap; +QHash httpCacheClientInvalidResponseCountMap; +QHash wsCacheClientConnectFailedCountMap; +QHash wsCacheClientInvalidResponseCountMap; + +static int gWorkersCount; + +///////////////////////////////////////////////////////////////////////////////////// + class ZhttpManager::Private : public QObject { Q_OBJECT @@ -63,7 +174,9 @@ class ZhttpManager::Private : public QObject { UnknownSession, HttpSession, - WebSocketSession + WebSocketSession, + CacheRequest, + CacheResponse }; class KeepAliveRegistration @@ -82,17 +195,17 @@ class ZhttpManager::Private : public QObject QStringList server_in_specs; QStringList server_in_stream_specs; QStringList server_out_specs; - QZmq::Socket *client_out_sock; - QZmq::Socket *client_out_stream_sock; - QZmq::Socket *client_in_sock; - QZmq::Socket *client_req_sock; - QZmq::Socket *server_in_sock; - QZmq::Socket *server_in_stream_sock; - QZmq::Socket *server_out_sock; - QZmq::Valve *client_in_valve; - QZmq::Valve *client_out_stream_valve; - QZmq::Valve *server_in_valve; - QZmq::Valve *server_in_stream_valve; + std::unique_ptr client_out_sock; + std::unique_ptr client_out_stream_sock; + std::unique_ptr client_in_sock; + std::unique_ptr client_req_sock; + std::unique_ptr server_in_sock; + std::unique_ptr server_in_stream_sock; + std::unique_ptr server_out_sock; + std::unique_ptr client_in_valve; + std::unique_ptr client_out_stream_valve; + std::unique_ptr server_in_valve; + std::unique_ptr server_in_stream_valve; QByteArray instanceId; int ipcFileMode; bool doBind; @@ -102,7 +215,7 @@ class ZhttpManager::Private : public QObject QHash clientSocksByRid; QHash serverSocksByRid; QList serverPendingSocks; - std::unique_ptr refreshTimer; + std::unique_ptr refreshTimer; QHash keepAliveRegistrations; QSet sessionRefreshBuckets[ZHTTP_REFRESH_BUCKETS]; int currentSessionRefreshBucket; @@ -119,22 +232,11 @@ class ZhttpManager::Private : public QObject Private(ZhttpManager *_q) : QObject(_q), q(_q), - client_out_sock(0), - client_out_stream_sock(0), - client_in_sock(0), - client_req_sock(0), - server_in_sock(0), - server_in_stream_sock(0), - server_out_sock(0), - client_in_valve(0), - client_out_stream_valve(0), - server_in_valve(0), - server_in_stream_valve(0), ipcFileMode(-1), doBind(false), currentSessionRefreshBucket(0) { - refreshTimer = std::make_unique(); + refreshTimer = std::make_unique(); refreshTimerConnection = refreshTimer->timeout.connect(boost::bind(&Private::refresh_timeout, this)); } @@ -165,17 +267,17 @@ class ZhttpManager::Private : public QObject { cosConnection.disconnect(); rrConnection.disconnect(); - delete client_req_sock; - delete client_out_sock; + client_req_sock.reset(); + client_out_sock.reset(); - client_out_sock = new QZmq::Socket(QZmq::Socket::Push, this); + client_out_sock = std::make_unique(QZmq::Socket::Push); cosConnection = client_out_sock->messagesWritten.connect(boost::bind(&Private::client_out_messagesWritten, this, boost::placeholders::_1)); client_out_sock->setHwm(OUT_HWM); client_out_sock->setShutdownWaitTime(CLIENT_WAIT_TIME); QString errorMessage; - if(!ZUtil::setupSocket(client_out_sock, client_out_specs, doBind, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(client_out_sock.get(), client_out_specs, doBind, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; @@ -188,11 +290,11 @@ class ZhttpManager::Private : public QObject { rrConnection.disconnect(); cossConnection.disconnect(); - delete client_req_sock; - delete client_out_stream_valve; - delete client_out_stream_sock; + client_req_sock.reset(); + client_out_stream_valve.reset(); + client_out_stream_sock.reset(); - client_out_stream_sock = new QZmq::Socket(QZmq::Socket::Router, this); + client_out_stream_sock = std::make_unique(QZmq::Socket::Router); cossConnection = client_out_stream_sock->messagesWritten.connect(boost::bind(&Private::client_out_stream_messagesWritten, this, boost::placeholders::_1)); client_out_stream_sock->setIdentity(instanceId); @@ -202,13 +304,13 @@ class ZhttpManager::Private : public QObject client_out_stream_sock->setImmediateEnabled(true); QString errorMessage; - if(!ZUtil::setupSocket(client_out_stream_sock, client_out_stream_specs, doBind, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(client_out_stream_sock.get(), client_out_stream_specs, doBind, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } - client_out_stream_valve = new QZmq::Valve(client_out_stream_sock, this); + client_out_stream_valve = std::make_unique(client_out_stream_sock.get()); clientOutStreamConnection = client_out_stream_valve->readyRead.connect(boost::bind(&Private::client_out_stream_readyRead, this, boost::placeholders::_1)); client_out_stream_valve->open(); @@ -219,24 +321,24 @@ class ZhttpManager::Private : public QObject bool setupClientIn() { rrConnection.disconnect(); - delete client_req_sock; - delete client_in_valve; - delete client_in_sock; + client_req_sock.reset(); + client_in_valve.reset(); + client_in_sock.reset(); - client_in_sock = new QZmq::Socket(QZmq::Socket::Sub, this); + client_in_sock = std::make_unique(QZmq::Socket::Sub); client_in_sock->setHwm(DEFAULT_HWM); client_in_sock->setShutdownWaitTime(0); client_in_sock->subscribe(instanceId + ' '); QString errorMessage; - if(!ZUtil::setupSocket(client_in_sock, client_in_specs, doBind, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(client_in_sock.get(), client_in_specs, doBind, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } - client_in_valve = new QZmq::Valve(client_in_sock, this); + client_in_valve = std::make_unique(client_in_sock.get()); clientConnection = client_in_valve->readyRead.connect(boost::bind(&Private::client_in_readyRead, this, boost::placeholders::_1)); client_in_valve->open(); @@ -248,18 +350,18 @@ class ZhttpManager::Private : public QObject { cosConnection.disconnect(); cossConnection.disconnect(); - delete client_out_sock; - delete client_out_stream_sock; - delete client_in_sock; + client_out_sock.reset(); + client_out_stream_sock.reset(); + client_in_sock.reset(); - client_req_sock = new QZmq::Socket(QZmq::Socket::Dealer, this); + client_req_sock = std::make_unique(QZmq::Socket::Dealer); rrConnection = client_req_sock->readyRead.connect(boost::bind(&Private::client_req_readyRead, this)); client_req_sock->setHwm(OUT_HWM); client_req_sock->setShutdownWaitTime(CLIENT_WAIT_TIME); QString errorMessage; - if(!ZUtil::setupSocket(client_req_sock, client_req_specs, doBind, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(client_req_sock.get(), client_req_specs, doBind, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; @@ -270,21 +372,21 @@ class ZhttpManager::Private : public QObject bool setupServerIn() { - delete server_in_valve; - delete server_in_sock; + server_in_valve.reset(); + server_in_sock.reset(); - server_in_sock = new QZmq::Socket(QZmq::Socket::Pull, this); + server_in_sock = std::make_unique(QZmq::Socket::Pull); server_in_sock->setHwm(IN_HWM); QString errorMessage; - if(!ZUtil::setupSocket(server_in_sock, server_in_specs, doBind, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(server_in_sock.get(), server_in_specs, doBind, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } - server_in_valve = new QZmq::Valve(server_in_sock, this); + server_in_valve = std::make_unique(server_in_sock.get()); serverConnection = server_in_valve->readyRead.connect(boost::bind(&Private::server_in_readyRead, this, boost::placeholders::_1)); server_in_valve->open(); @@ -295,21 +397,21 @@ class ZhttpManager::Private : public QObject bool setupServerInStream() { serverStreamConnection.disconnect(); - delete server_in_stream_sock; + server_in_stream_sock.reset(); - server_in_stream_sock = new QZmq::Socket(QZmq::Socket::Router, this); + server_in_stream_sock = std::make_unique(QZmq::Socket::Router); server_in_stream_sock->setIdentity(instanceId); server_in_stream_sock->setHwm(DEFAULT_HWM); QString errorMessage; - if(!ZUtil::setupSocket(server_in_stream_sock, server_in_stream_specs, doBind, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(server_in_stream_sock.get(), server_in_stream_specs, doBind, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } - server_in_stream_valve = new QZmq::Valve(server_in_stream_sock, this); + server_in_stream_valve = std::make_unique(server_in_stream_sock.get()); serverStreamConnection = server_in_stream_valve->readyRead.connect(boost::bind(&Private::server_in_stream_readyRead, this, boost::placeholders::_1)); server_in_stream_valve->open(); @@ -320,9 +422,9 @@ class ZhttpManager::Private : public QObject bool setupServerOut() { sosConnection.disconnect(); - delete server_out_sock; + server_out_sock.reset(); - server_out_sock = new QZmq::Socket(QZmq::Socket::Pub, this); + server_out_sock = std::make_unique(QZmq::Socket::Pub); sosConnection = server_out_sock->messagesWritten.connect(boost::bind(&Private::server_out_messagesWritten, this, boost::placeholders::_1)); server_out_sock->setWriteQueueEnabled(false); @@ -330,7 +432,7 @@ class ZhttpManager::Private : public QObject server_out_sock->setShutdownWaitTime(SERVER_WAIT_TIME); QString errorMessage; - if(!ZUtil::setupSocket(server_out_sock, server_out_specs, doBind, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(server_out_sock.get(), server_out_specs, doBind, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; @@ -356,19 +458,55 @@ class ZhttpManager::Private : public QObject return best; } - void tryRespondCancel(SessionType type, const QByteArray &id, const ZhttpRequestPacket &packet) + void send_response_to_client( + ZhttpResponsePacket::Type packetType, + const QByteArray &clientId, + const QByteArray &from, + int credits = 0, + ZhttpResponsePacket *responsePacket = NULL, + const QByteArray &responseKey = NULL) { - assert(!packet.from.isEmpty()); + assert(!from.isEmpty()); + + ZhttpResponsePacket out; + + ZhttpResponsePacket::Id tempId; + + QByteArray newFrom = from; - // if this was not an error packet, send cancel - if(packet.type != ZhttpRequestPacket::Error && packet.type != ZhttpRequestPacket::Cancel) + switch (packetType) { - ZhttpResponsePacket out; - out.from = instanceId; - out.ids += ZhttpResponsePacket::Id(id); - out.type = ZhttpResponsePacket::Cancel; - write(type, out, packet.from); + case ZhttpResponsePacket::Data: + if (responsePacket != NULL) + { + out = *responsePacket; + out.ids[0].id = clientId; + if (responseKey != NULL) + { + out.headers.removeAll("sec-websocket-accept"); + out.headers += HttpHeader("sec-websocket-accept", responseKey); + } + + // update the counter for prometheus + count_responses(responseKey != NULL ? "WS_INIT" : "WS"); + break; + } + return; + case ZhttpResponsePacket::Credit: + tempId.id = clientId; + out.ids += tempId; + out.type = packetType; + out.credits = credits; + break; + default: + tempId.id = clientId; + out.ids += tempId; + out.type = packetType; + break; } + + out.from = instanceId;//clientInstanceId; + writeToClient(CacheResponse, out, newFrom); } void write(SessionType type, const ZhttpRequestPacket &packet) @@ -381,15 +519,15 @@ class ZhttpManager::Private : public QObject if(client_out_sock) { - if(log_outputLevel() >= LOG_LEVEL_DEBUG) - LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s client: OUT", logprefix); + if(log_outputLevel() > LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s client: OUT1", logprefix); client_out_sock->write(QList() << buf); } else { - if(log_outputLevel() >= LOG_LEVEL_DEBUG) - LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s client req: OUT", logprefix); + if(log_outputLevel() > LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s client req: OUT2", logprefix); client_req_sock->write(QList() << QByteArray() << buf); } @@ -403,8 +541,8 @@ class ZhttpManager::Private : public QObject QVariant vpacket = packet.toVariant(); QByteArray buf = QByteArray("T") + TnetString::fromVariant(vpacket); - if(log_outputLevel() >= LOG_LEVEL_DEBUG) - LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s client: OUT %s", logprefix, instanceAddress.data()); + if(log_outputLevel() > LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s client: OUT3 %s", logprefix, instanceAddress.data()); QList msg; msg += instanceAddress; @@ -418,7 +556,130 @@ class ZhttpManager::Private : public QObject assert(server_out_sock); const char *logprefix = logPrefixForType(type); - QVariant vpacket = packet.toVariant(); + QByteArray packetId = packet.ids.first().id; + + // cache process + if (gCacheEnable == true && type != SessionType::CacheRequest && type != SessionType::CacheResponse) + { + pause_cache_thread(); + + int ccIndex = get_cc_index_from_clientId(packetId); + // update data receive time + if (ccIndex >= 0) + { + gWsCacheClientList[ccIndex].lastResponseTime = QDateTime::currentMSecsSinceEpoch(); + } + + if (packet.code == 101) // ws client init response code + { + if (ccIndex >= 0) + { + // cache client + gWsCacheClientList[ccIndex].initFlag = true; + gWsCacheClientList[ccIndex].lastResponseTime = QDateTime::currentMSecsSinceEpoch(); + gWsCacheClientList[ccIndex].lastResponseSeq = -1; + gWsCacheClientList[ccIndex].receiver = packet.from; + log_debug("[WS] Initialized Cache client%d, %s, from=%s", ccIndex, gWsCacheClientList[ccIndex].clientId.data(), + packet.from.toHex().data()); + gWsInitResponsePacket = packet; + } + else + { + // real client + log_debug("[WS] Initialized real client=%s", packetId.data()); + } + } + else + { + switch (packet.type) + { + case ZhttpResponsePacket::Cancel: + case ZhttpResponsePacket::Close: + case ZhttpResponsePacket::Error: + { + log_debug("[WS] switching client of response error, condition=%s", packet.condition.data()); + + // get error type + QString conditionStr = QString(packet.condition); + if (conditionStr.compare("remote-connection-failed", Qt::CaseInsensitive) == 0 || + conditionStr.compare("connection-timeout", Qt::CaseInsensitive) == 0) + { + log_debug("[WS] Sleeping for 10 seconds"); + sleep(10); + } + + // if cache client0 is ON, start cache client1 + int ccIndex = get_cc_index_from_clientId(packetId); + if (ccIndex >= 0) + { + log_debug("[WS] disabled cache client %d", ccIndex); + QString urlPath = gWsBackendUrlList[ccIndex]; + wsCacheClientConnectFailedCountMap[urlPath]++; + gWsCacheClientList[ccIndex].initFlag = false; + } + } + break; + case ZhttpResponsePacket::Credit: + log_debug("[WS] passing credit response"); + break; + case ZhttpResponsePacket::Ping: + log_debug("[WS] passing ping response"); + break; + case ZhttpResponsePacket::KeepAlive: + log_debug("[WS] passing keep-alive response"); + break; + case ZhttpResponsePacket::Data: + if (ccIndex >= 0) + { + // increase credit + int creditSize = static_cast(packet.body.size()); + ZhttpRequestPacket out; + out.type = ZhttpRequestPacket::Credit; + out.credits = creditSize; + send_ws_request_over_cacheclient(out, NULL, ccIndex); + + process_ws_cacheclient_response(packet, ccIndex, instanceAddress); + resume_cache_thread(); + return; + } + else + { + /* + // increase credit + int creditSize = static_cast(packet.body.size()); + ZhttpRequestPacket out; + out.type = ZhttpRequestPacket::Credit; + out.credits = creditSize; + ZhttpRequestPacket::Id tempId; + tempId.id = packetId; // id + tempId.seq = update_request_seq(packetId); + out.ids += tempId; + ZhttpRequest *req = serverReqsByRid.value(ZhttpRequest::Rid(packet.from, packetId)); + if(req) + { + req->handle(packetId, tempId.seq, out); + } + */ + int ret = process_http_response(packet, instanceAddress); + if (ret < 0) + { + resume_cache_thread(); + return; + } + } + break; + default: + break; + } + } + + resume_cache_thread(); + } + + ZhttpResponsePacket p = packet; + + p.ids.first().seq = get_client_new_response_seq(packetId); + QVariant vpacket = p.toVariant(); QByteArray buf = instanceAddress + " T" + TnetString::fromVariant(vpacket); if(log_outputLevel() >= LOG_LEVEL_DEBUG) @@ -427,12 +688,407 @@ class ZhttpManager::Private : public QObject server_out_sock->write(QList() << buf); } + void writeToClient1(SessionType type, ZhttpResponsePacket &packet, const QByteArray &instanceAddress) + { + assert(server_out_sock); + const char *logprefix = logPrefixForType(type); + + QByteArray clientId = packet.ids.first().id; + int newSeq = get_client_new_response_seq(clientId); + if (newSeq < 0) + { + log_debug("[WS] failed to get new response seq %s", clientId.constData()); + return; + } + packet.ids.first().seq = newSeq; + + QVariant vpacket = packet.toVariant(); + QByteArray buf = instanceAddress + " T" + TnetString::fromVariant(vpacket); + + if(log_outputLevel() >= LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s server: OUT %s", logprefix, instanceAddress.data()); + + server_out_sock->write(QList() << buf); + } + + void writeToClient(SessionType type, ZhttpResponsePacket &packet, const QByteArray &instanceAddress) + { + assert(server_out_sock); + const char *logprefix = logPrefixForType(type); + + QByteArray clientId = packet.ids.first().id; + + QByteArray packetBody = packet.body; + + if (packetBody.isEmpty()) + { + int newSeq = get_client_new_response_seq(clientId); + if (newSeq < 0) + { + log_debug("[WS] failed to get new response seq %s", clientId.constData()); + return; + } + packet.ids.first().seq = newSeq; + + QVariant vpacket = packet.toVariant(); + QByteArray buf = instanceAddress + " T" + TnetString::fromVariant(vpacket); + + if(log_outputLevel() >= LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s server: OUT %s", logprefix, instanceAddress.data()); + + server_out_sock->write(QList() << buf); + } + else + { + int clientCreditSize = get_client_credit_size(clientId); + log_debug("%d", clientCreditSize); + + QList chunks; + int offset = 0; + while (offset < packetBody.size()) + { + int len = qMin(clientCreditSize, packetBody.size() - offset); + chunks << packetBody.mid(offset, len); + offset += len; + } + + log_debug("%d", chunks.size()); + + for (int i=0; i= LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s server: OUT %s", logprefix, instanceAddress.data()); + + server_out_sock->write(QList() << buf); + QThread::usleep(1); + } + } + } + + void writeToClient1_(const QByteArray &cacheItemId, const QByteArray &clientId, const QString &msgId, const QByteArray &instanceAddress, const QByteArray &instanceId) + { + assert(server_out_sock); + + int newSeq = get_client_new_response_seq(clientId); + if (newSeq < 0) + { + log_debug("[WS] failed_ to get new response seq %s", clientId.constData()); + return; + } + + QByteArray buf = load_cache_response_buffer(instanceAddress, cacheItemId, clientId, newSeq, msgId, instanceId, 0); + + // count methods + numMessageSent++; + + server_out_sock->write(QList() << buf); + QThread::usleep(1); + } + + void writeToClient2_(const QByteArray &cacheItemId, const QByteArray &clientId, const QString &msgId, const QByteArray &instanceAddress, const QByteArray &instanceId) + { + assert(server_out_sock); + + CacheItem* pCacheItem = load_cache_item(cacheItemId); + + int newSeq = get_client_new_response_seq(clientId); + + if (newSeq < 0) + { + log_debug("[WS] failed_ to get new response seq %s", clientId.constData()); + return; + } + + QByteArray data; + if (pCacheItem != NULL && pCacheItem->proto == Scheme::http) + { + data = load_cache_response_buffer(instanceAddress, cacheItemId, clientId, newSeq, msgId, instanceId, 0); + } + else + { + data = load_cache_response_buffer(instanceAddress, cacheItemId, clientId, newSeq, msgId, instanceId, 0); + } + + server_out_sock->write(QList() << data); + QThread::usleep(1); +/* + if (newSeq < 0) + { + log_debug("[WS] failed_ to get new response seq %s", clientId.constData()); + return; + } + + QByteArray buf = load_cache_response_buffer(instanceAddress, cacheItemId, clientId, newSeq, msgId, instanceId, 0); + + // count methods + numMessageSent++; + + server_out_sock->write(QList() << buf); + QThread::usleep(1); +*/ + } + + void writeToClient_(const QByteArray &cacheItemId, const QByteArray &clientId, const QString &msgId, const QByteArray &instanceAddress, const QByteArray &instanceId) + { + assert(server_out_sock); + const char *logprefix = logPrefixForType(CacheResponse); + + CacheItem* pCacheItem = load_cache_item(cacheItemId); + + QByteArray data; + if (pCacheItem != NULL && pCacheItem->proto == Scheme::http) + { + int newSeq = get_client_new_response_seq(clientId); + data = load_cache_response_buffer(instanceAddress, cacheItemId, clientId, newSeq, msgId, instanceId, 0); + } + else + { + data = load_cache_response_buffer(instanceAddress, cacheItemId, clientId, 0, msgId, instanceId, 0); + } + + // count methods + numMessageSent++; + + // extract body + QByteArray packetBody = ""; + QByteArray bodyKey = "4:body,"; + int bodyPos = data.indexOf(bodyKey); + if (bodyPos != -1) + { + int lengthStart = bodyPos + bodyKey.size(); + int colonPos = data.indexOf(':', lengthStart); + if (colonPos != -1) + { + int bodyLength = data.mid(lengthStart, colonPos - lengthStart).toInt(); + packetBody = data.mid(colonPos + 1, bodyLength); + + if (pCacheItem != NULL && pCacheItem->proto == Scheme::http) + { + // check more:true flag + QByteArray moreTrueKey = "4:more,4:true!"; + int moreTruePos = data.indexOf(moreTrueKey); + + int removeLength = (colonPos + 1 + bodyLength) - bodyPos + 1; // 1 is for ',' + if (moreTruePos != -1) + { + // Remove "4:body,:" + data.remove(bodyPos, removeLength); + } + else + { + // check more:false flag + QByteArray moreFalseKey = "4:more,4:false!"; + int moreFalsePos = data.indexOf(moreFalseKey); + if (moreFalsePos != -1) + { + // Remove moreFalseKey + data.remove(moreFalsePos, moreFalseKey.size()); + removeLength += moreFalseKey.size(); + } + + // Replace "4:body,: with moreTrueKey" + data.replace(bodyPos, removeLength, moreTrueKey); + removeLength -= moreTrueKey.size(); + } + + // update T-length + int tPos = data.indexOf("T"); + if (tPos != -1) + { + tPos += 1; + colonPos = data.indexOf(':', tPos); + if (colonPos != -1) + { + int tLength = data.mid(tPos, colonPos-tPos).toInt(); + if (tLength > 0) + { + int newLength = tLength - removeLength; + QByteArray newHeader = QByteArray::number(newLength); + data.replace(tPos, colonPos-tPos, newHeader); + } + } + } + } + } + } + + // send first part of response if http + if (pCacheItem != NULL && pCacheItem->proto == Scheme::http) + { + server_out_sock->write(QList() << data); + } + + if (packetBody.isEmpty()) + { + int newSeq = get_client_new_response_seq(clientId); + if (newSeq < 0) + { + log_debug("[WS] failed_ to get new response seq %s", clientId.constData()); + return; + } + + server_out_sock->write(QList() << data); + QThread::usleep(1); + } + else + { + int clientCreditSize = get_client_credit_size(clientId); + + QList chunks; + int offset = 0; + while (offset < packetBody.size()) + { + int len = qMin(clientCreditSize, packetBody.size() - offset); + chunks << packetBody.mid(offset, len); + offset += len; + } + + for (int i=0; i= LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s server: OUT %s", logprefix, instanceAddress.data()); + + server_out_sock->write(QList() << buf); + QThread::usleep(1); + } + } + + //server_out_sock->write(QList() << buf); + //QThread::usleep(1); + } + + void writeToClient__(SessionType type, ZhttpResponsePacket &packet, const QByteArray &clientId, const QByteArray &instanceAddress, const QByteArray &instanceId) + { + assert(server_out_sock); + const char *logprefix = logPrefixForType(type); + + // count methods + numMessageSent++; + + //server_out_sock->write(QList() << buf); + QByteArray packetBody = packet.body; + + if (packetBody.isEmpty()) + { + packet.ids.first().id = clientId; + int newSeq = get_client_new_response_seq(clientId); + if (newSeq < 0) + { + log_debug("[WS] failed to get new response seq %s", clientId.constData()); + return; + } + packet.ids.first().seq = newSeq; + packet.from = instanceId; + + QVariant vpacket = packet.toVariant(); + QByteArray buf = instanceAddress + " T" + TnetString::fromVariant(vpacket); + + if(log_outputLevel() >= LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s server: OUT %s", logprefix, instanceAddress.data()); + + server_out_sock->write(QList() << buf); + } + else + { + int clientCreditSize = get_client_credit_size(clientId); + + QList chunks; + int offset = 0; + while (offset < packetBody.size()) + { + int len = qMin(clientCreditSize, packetBody.size() - offset); + chunks << packetBody.mid(offset, len); + offset += len; + } + + for (int i=0; i= LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s server: OUT %s", logprefix, instanceAddress.data()); + + server_out_sock->write(QList() << buf); + QThread::usleep(1); + } + } + } + + void writeToClient1__(SessionType type, ZhttpResponsePacket &packet, const QByteArray &clientId, const QByteArray &instanceAddress, const QByteArray &instanceId) + { + assert(server_out_sock); + const char *logprefix = logPrefixForType(type); + + packet.ids.first().id = clientId; + int newSeq = get_client_new_response_seq(clientId); + if (newSeq < 0) + { + log_debug("[WS] failed to get new response seq %s", clientId.constData()); + return; + } + packet.ids.first().seq = newSeq; + packet.from = instanceId; + + QVariant vpacket = packet.toVariant(); + QByteArray buf = instanceAddress + " T" + TnetString::fromVariant(vpacket); + + if(log_outputLevel() >= LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s server: OUT %s", logprefix, instanceAddress.data()); + + // count methods + numMessageSent++; + + server_out_sock->write(QList() << buf); + } + static const char *logPrefixForType(SessionType type) { switch(type) { case HttpSession: return "zhttp"; case WebSocketSession: return "zws"; + case CacheRequest: return "cache-request"; + case CacheResponse: return "cache-response"; default: return "zhttp/zws"; } } @@ -496,7 +1152,10 @@ class ZhttpManager::Private : public QObject zresp.from = instanceId; zresp.ids = ids; zresp.type = ZhttpResponsePacket::KeepAlive; - write(type, zresp, zhttpAddress); + if (gCacheEnable == true) + writeToClient(type, zresp, zhttpAddress); + else + write(type, zresp, zhttpAddress); } void client_out_messagesWritten(int count) @@ -516,7 +1175,7 @@ class ZhttpManager::Private : public QObject void client_req_readyRead() { - QPointer self = this; + std::weak_ptr self = q->d; while(client_req_sock->canRead()) { @@ -563,7 +1222,7 @@ class ZhttpManager::Private : public QObject if(req) { req->handle(id.id, id.seq, p); - if(!self) + if(self.expired()) return; continue; @@ -605,7 +1264,7 @@ class ZhttpManager::Private : public QObject return; } - QPointer self = this; + std::weak_ptr self = q->d; foreach(const ZhttpResponsePacket::Id &id, p.ids) { @@ -614,7 +1273,7 @@ class ZhttpManager::Private : public QObject if(sock) { sock->handle(id.id, id.seq, p); - if(!self) + if(self.expired()) return; continue; @@ -625,7 +1284,7 @@ class ZhttpManager::Private : public QObject if(req) { req->handle(id.id, id.seq, p); - if(!self) + if(self.expired()) return; continue; @@ -720,10 +1379,59 @@ class ZhttpManager::Private : public QObject if(sock) { log_warning("zws server: received message for existing request id, canceling"); - tryRespondCancel(WebSocketSession, id.id, p); + if(p.type != ZhttpRequestPacket::Error && p.type != ZhttpRequestPacket::Cancel) + send_response_to_client(ZhttpResponsePacket::Cancel, id.id, p.from); return; } + if (gCacheEnable == true) + { + pause_cache_thread(); + + // if requests from cache client + int ccIndex = get_cc_index_from_init_request(p); + if (ccIndex >= 0 && ccIndex < gWsCacheClientList.count()) + { + gWsCacheClientList[ccIndex].initFlag = false; + gWsCacheClientList[ccIndex].clientId = id.id; + gWsCacheClientList[ccIndex].instanceId = instanceId; + gWsCacheClientList[ccIndex].msgIdCount = -1; + gWsCacheClientList[ccIndex].from = p.from; + gWsCacheClientList[ccIndex].lastRequestSeq = id.seq; + gWsCacheClientList[ccIndex].lastResponseSeq = -1; + gWsCacheClientList[ccIndex].lastRequestTime = QDateTime::currentMSecsSinceEpoch(); + + log_debug("[WS] Registered new cache %d client=%s, from=%s, instanceId=%s", + ccIndex, id.id.constData(), p.from.constData(), instanceId.constData()); + } + else // if request from real client + { + log_debug("[WS] received init request from real client"); + if (get_main_cc_index(instanceId) < 0) + { + log_warning("[WS] not initialized cache client, ignore"); + if(p.type != ZhttpRequestPacket::Error && p.type != ZhttpRequestPacket::Cancel) + send_response_to_client(ZhttpResponsePacket::Cancel, id.id, p.from); + resume_cache_thread(); + return; + } + else + { + // get resp key + QByteArray responseKey = calculate_response_seckey_from_init_request(p); + int creditSize = p.credits; + // register ws client + register_ws_client(id.id, p.from, p.uri.toString(), creditSize); + // respond with cached init packet + send_response_to_client(ZhttpResponsePacket::Data, id.id, p.from, 0, &gWsInitResponsePacket, responseKey); + resume_cache_thread(); + return; + } + } + + resume_cache_thread(); + } + sock = new ZWebSocket; if(!sock->setupServer(q, id.id, id.seq, p)) { @@ -747,16 +1455,43 @@ class ZhttpManager::Private : public QObject if(req) { log_warning("zhttp server: received message for existing request id, canceling"); - tryRespondCancel(HttpSession, id.id, p); + if(p.type != ZhttpRequestPacket::Error && p.type != ZhttpRequestPacket::Cancel) + send_response_to_client(ZhttpResponsePacket::Cancel, id.id, p.from); return; } - req = new ZhttpRequest; - if(!req->setupServer(q, id.id, id.seq, p)) + // cache process + if (gCacheEnable == true) { - delete req; - return; - } + pause_cache_thread(); + + if (!p.headers.contains(HTTP_REFRESH_HEADER)) + { + register_http_client(id.id, p.from, p.uri.toString(), p.credits); + } + else + { + QString tmpStr = QString::fromUtf8(p.headers.get(HTTP_REFRESH_HEADER)); + QByteArray msgIdByte = QByteArray::fromHex(qPrintable(tmpStr.remove('\"'))); + CacheItem *pCacheItem = load_cache_item(msgIdByte); + if (pCacheItem != NULL) + { + pCacheItem->requestPacket.ids[0].id = id.id; + //store_cache_item_field(msgIdByte, "requestPacket", TnetString::fromVariant(pCacheItem->requestPacket.toVariant())); + } + // remove HTTP_REFRESH_HEADER header + p.headers.removeAll(HTTP_REFRESH_HEADER); + } + + resume_cache_thread(); + } + + req = new ZhttpRequest; + if(!req->setupServer(q, id.id, id.seq, p)) + { + delete req; + return; + } serverReqsByRid.insert(rid, req); serverPendingReqs += req; @@ -769,7 +1504,8 @@ class ZhttpManager::Private : public QObject else { log_debug("zhttp/zws server: rejecting unsupported scheme: %s", qPrintable(p.uri.scheme())); - tryRespondCancel(UnknownSession, id.id, p); + if(p.type != ZhttpRequestPacket::Error && p.type != ZhttpRequestPacket::Cancel) + send_response_to_client(ZhttpResponsePacket::Cancel, id.id, p.from); return; } } @@ -801,194 +1537,1705 @@ class ZhttpManager::Private : public QObject ZhttpRequestPacket p; if(!p.fromVariant(data)) { - log_warning("zhttp/zws server: received message with invalid format (parse failed), skipping"); - return; + log_warning("zhttp/zws server: received message with invalid format (parse failed), skipping"); + return; + } + + std::weak_ptr self = q->d; + + for (int i=0; i(p.body.size())); + process_ws_stream_request(packetId, p); + break; + default: + break; + } + + resume_cache_thread(); + continue; + } + else // cache client + { + switch (p.type) + { + case ZhttpRequestPacket::Cancel: + case ZhttpRequestPacket::Close: + case ZhttpRequestPacket::Error: + { + log_debug("[WS] switching client of request error, condition=%s", p.condition.data()); + + // get error type + QString conditionStr = QString(p.condition); + if (conditionStr.compare("remote-connection-failed", Qt::CaseInsensitive) == 0 || + conditionStr.compare("connection-timeout", Qt::CaseInsensitive) == 0) + { + log_debug("[WS] Sleeping for 10 seconds"); + sleep(10); + } + + // if cache client0 is ON, start cache client1 + int ccIndex = get_cc_index_from_clientId(packetId); + if (ccIndex >= 0) + { + log_debug("[WS] disabled cache client %d", ccIndex); + QString urlPath = gWsBackendUrlList[ccIndex]; + wsCacheClientConnectFailedCountMap[urlPath]++; + gWsCacheClientList[ccIndex].initFlag = false; + } + } + break; + default: + break; + } + } + + resume_cache_thread(); + + newSeq = update_request_seq(packetId); + if (newSeq >= 0) + p.ids[i].seq = newSeq; + else + newSeq = p.ids[i].seq; + } + + // is this for a websocket? + ZWebSocket *sock = serverSocksByRid.value(ZWebSocket::Rid(p.from, packetId)); + if(sock) + { + sock->handle(packetId, newSeq, p); + if(self.expired()) + return; + + continue; + } + + // is this for an http request? + ZhttpRequest *req = serverReqsByRid.value(ZhttpRequest::Rid(p.from, packetId)); + if(req) + { + req->handle(packetId, newSeq, p); + if(self.expired()) + return; + + continue; + } + + log_debug("zhttp/zws server: received message for unknown request id, skipping"); + } + } + + void refresh_timeout() + { + QHash > clientSessionsBySender[2]; // index corresponds to type + QHash > serverSessionsBySender[2]; // index corresponds to type + + // process the current bucket + const QSet &bucket = sessionRefreshBuckets[currentSessionRefreshBucket]; + foreach(KeepAliveRegistration *r, bucket) + { + QPair rid; + bool isServer; + if(r->type == HttpSession) + { + rid = r->p.req->rid(); + isServer = r->p.req->isServer(); + } + else // WebSocketSession + { + rid = r->p.sock->rid(); + isServer = r->p.sock->isServer(); + } + + QByteArray sender; + if(isServer) + { + sender = rid.first; + } + else + { + if(r->type == HttpSession) + sender = r->p.req->toAddress(); + else // WebSocketSession + sender = r->p.sock->toAddress(); + } + + assert(!sender.isEmpty()); + + QHash > &sessionsBySender = (isServer ? serverSessionsBySender[r->type - 1] : clientSessionsBySender[r->type - 1]); + + if(!sessionsBySender.contains(sender)) + sessionsBySender.insert(sender, QList()); + + QList &sessions = sessionsBySender[sender]; + sessions += r; + + // if we're at max, send out now + if(sessions.count() >= ZHTTP_IDS_MAX) + { + if(isServer) + { + QList ids; + foreach(KeepAliveRegistration *i, sessions) + { + assert(i->type == r->type); + if(r->type == HttpSession) + ids += ZhttpResponsePacket::Id(i->p.req->rid().second, i->p.req->outSeqInc()); + else // WebSocketSession + ids += ZhttpResponsePacket::Id(i->p.sock->rid().second, i->p.sock->outSeqInc()); + } + + writeKeepAlive(r->type, ids, sender); + } + else + { + QList ids; + foreach(KeepAliveRegistration *i, sessions) + { + assert(i->type == r->type); + if(r->type == HttpSession) + ids += ZhttpRequestPacket::Id(i->p.req->rid().second, i->p.req->outSeqInc()); + else // WebSocketSession + ids += ZhttpRequestPacket::Id(i->p.sock->rid().second, i->p.sock->outSeqInc()); + } + + writeKeepAlive(r->type, ids, sender); + } + + sessions.clear(); + sessionsBySender.remove(sender); + } + } + + // send last packets + for(int n = 0; n < 2; ++n) + { + SessionType type = (SessionType)(n + 1); + + { + QHashIterator > sit(clientSessionsBySender[n]); + while(sit.hasNext()) + { + sit.next(); + const QByteArray &sender = sit.key(); + const QList &sessions = sit.value(); + + if(!sessions.isEmpty()) + { + QList ids; + foreach(KeepAliveRegistration *i, sessions) + { + assert(i->type == type); + if(type == HttpSession) + ids += ZhttpRequestPacket::Id(i->p.req->rid().second, i->p.req->outSeqInc()); + else // WebSocketSession + ids += ZhttpRequestPacket::Id(i->p.sock->rid().second, i->p.sock->outSeqInc()); + } + + writeKeepAlive(type, ids, sender); + } + } + } + + { + QHashIterator > sit(serverSessionsBySender[n]); + while(sit.hasNext()) + { + sit.next(); + const QByteArray &sender = sit.key(); + const QList &sessions = sit.value(); + + if(!sessions.isEmpty()) + { + QList ids; + foreach(KeepAliveRegistration *i, sessions) + { + assert(i->type == type); + if(type == HttpSession) + ids += ZhttpResponsePacket::Id(i->p.req->rid().second, i->p.req->outSeqInc()); + else // WebSocketSession + ids += ZhttpResponsePacket::Id(i->p.sock->rid().second, i->p.sock->outSeqInc()); + } + + writeKeepAlive(type, ids, sender); + } + } + } + } + + ++currentSessionRefreshBucket; + if(currentSessionRefreshBucket >= ZHTTP_REFRESH_BUCKETS) + currentSessionRefreshBucket = 0; + } + + void timer_send_ping_to_client(QByteArray clientId) + { + if (gWsClientMap.contains(clientId)) + { + log_debug("_[TIMER] send ping to client %s", clientId.data()); + send_response_to_client(ZhttpResponsePacket::Ping, clientId, gWsClientMap[clientId].from); + QTimer::singleShot(PING_INTERVAL * 1000, [=]() { + timer_send_ping_to_client(clientId); + }); + } + else + { + log_debug("_[TIMER] exit ping for client %s", clientId.data()); + } + } + + void refresh_cache(QByteArray itemId, QString urlPath, qint64 refreshCount) + { + log_debug("_[TIMER] cache refresh %s %s", itemId.toHex().data(), qPrintable(urlPath)); + CacheItem *pCacheItem = load_cache_item(itemId); + if (pCacheItem == NULL) + { + log_debug("_[TIMER] exit refresh %s", itemId.toHex().data()); + return; + } + + // check refresh count (can be duplicated by the removed cache item.) + if (refreshCount != pCacheItem->lastRefreshCount) + { + log_debug("_[TIMER] got invalid timer %s, expect %d, but %d", itemId.toHex().data(), refreshCount, pCacheItem->lastRefreshCount); + return; + } + pCacheItem->lastRefreshCount++; + refreshCount = pCacheItem->lastRefreshCount; + + int timeInterval = get_next_cache_refresh_interval(itemId); + qint64 currMTime = QDateTime::currentMSecsSinceEpoch(); + qint64 accessTimeoutMSeconds = gAccessTimeoutSeconds * 1000; + if (pCacheItem->cachedFlag == true) + { + // delete old cache items if it`s not auto_refresh_unerase + if ((pCacheItem->refreshFlag & AUTO_REFRESH_UNERASE) == 0) + { + qint64 accessDiff = currMTime - pCacheItem->lastAccessTime; + if (accessDiff > accessTimeoutMSeconds) + { + // remove cache item + log_debug("[CACHE] deleting cache item for access timeout %s", itemId.toHex().data()); + remove_cache_item(itemId); + return; + } + } + + if (timeInterval > 0) + { + if (pCacheItem->proto == Scheme::http) + { + QByteArray reqBody = pCacheItem->requestPacket.body; + QString newMsgId = QString("\"%1\"").arg(itemId.toHex().data()); + replace_id_field(reqBody, QString("__MSGID__"), newMsgId); + send_http_post_request_with_refresh_header(urlPath, reqBody, itemId.toHex().data()); + } + else if (pCacheItem->proto == Scheme::websocket) + { + // Send client cache request packet for auto-refresh + int ccIndex = get_cc_index_from_clientId(pCacheItem->cacheClientId); + pCacheItem->newMsgId = send_ws_request_over_cacheclient(pCacheItem->requestPacket, QString("__MSGID__"), ccIndex); + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + } + } + } + else + { + if (pCacheItem->retryCount > RETRY_RESPONSE_MAX_COUNT) + { + log_debug("[_TIMER] reached max retry count"); + return; + } + pCacheItem->retryCount++; + // switch backend of the failed response + if (pCacheItem->proto == Scheme::http) + { + qint64 accessDiff = currMTime - pCacheItem->lastAccessTime; + if (accessDiff > accessTimeoutMSeconds) + { + // prometheus status + if (httpCacheClientConnectFailedCountMap.contains(urlPath)) + httpCacheClientConnectFailedCountMap[urlPath]++; + } + + urlPath = get_switched_http_backend_url(urlPath); + for (int i=0; ihttpBackendNo = i; + //store_cache_item_field(itemId, "httpBackendNo", pCacheItem->httpBackendNo); + break; + } + } + + QByteArray reqBody = pCacheItem->requestPacket.body; + QString newMsgId = QString("\"%1\"").arg(itemId.toHex().data()); + replace_id_field(reqBody, QString("__MSGID__"), newMsgId); + send_http_post_request_with_refresh_header(urlPath, reqBody, itemId.toHex().data()); + } + else if (pCacheItem->proto == Scheme::websocket) + { + qint64 accessDiff = currMTime - pCacheItem->lastAccessTime; + if (accessDiff > accessTimeoutMSeconds) + { + // prometheus status + if (wsCacheClientConnectFailedCountMap.contains(urlPath)) + wsCacheClientConnectFailedCountMap[urlPath]++; + } + + // Send client cache request packet for auto-refresh + int ccIndex = get_cc_next_index_from_clientId(pCacheItem->cacheClientId, instanceId); + pCacheItem->cacheClientId = gWsCacheClientList[ccIndex].clientId; + urlPath = gWsCacheClientList[ccIndex].urlPath; + pCacheItem->newMsgId = send_ws_request_over_cacheclient(pCacheItem->requestPacket, QString("__MSGID__"), ccIndex); + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + } + } + + if (timeInterval > 0) + { + QTimer::singleShot(timeInterval * 1000, [=]() { + refresh_cache(itemId, urlPath, refreshCount); + }); + } + } + + void register_cache_refresh(QByteArray itemId, QString urlPath) + { + CacheItem *pCacheItem = load_cache_item(itemId); + if (pCacheItem == NULL) + { + log_debug("[REFRESH] Canceled cache item because it not exist %s", itemId.toHex().data()); + return; + } + + log_debug("[REFRESH] Registered new cache refresh %s, %s", itemId.toHex().data(), qPrintable(urlPath)); + + pCacheItem->lastRefreshCount = 0; + int timeInterval = get_next_cache_refresh_interval(itemId); + if (timeInterval > 0) + { + QTimer::singleShot(timeInterval * 1000, [=]() { + refresh_cache(itemId, urlPath, 0); + }); + } + //store_cache_item_field(itemId, "lastRefreshCount", 0); + } + + void scan_subscribe_update() + { + foreach(QByteArray itemId, get_cache_item_ids()) + { + CacheItem* pCacheItem = load_cache_item(itemId); + if (pCacheItem == NULL) + { + log_debug("[SCAN] not found cache item %s", itemId.toHex().data()); + continue; + } + if ((pCacheItem->cachedFlag == true) && (pCacheItem->proto == Scheme::none) && + (pCacheItem->methodType == SUBSCRIBE_METHOD)) + { + QByteArray updateCountKey = itemId + "-updateCount"; + QByteArray countBytes = redis_load_cache_response(updateCountKey); + int updateCount = countBytes.toInt(); + log_debug("[SCAN] %d, %d", pCacheItem->subscriptionUpdateCount, updateCount); + if (pCacheItem->subscriptionUpdateCount != updateCount) + { + pCacheItem->subscriptionUpdateCount = updateCount; + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + + QByteArray updateKey = itemId + "-update"; + QByteArray packetBuf = redis_load_cache_response(updateKey); + QVariant data = TnetString::toVariant(packetBuf); + if(!data.isNull()) + { + ZhttpResponsePacket p; + if(p.fromVariant(data)) + { + log_debug("[SUBSCRIBE] sending update to replica"); + // send update subscribe to all clients + QHash::iterator it = pCacheItem->clientMap.begin(); + while (it != pCacheItem->clientMap.end()) + { + QByteArray cliId = it.key(); + if (gWsClientMap.contains(cliId)) + { + QString clientMsgId = pCacheItem->clientMap[cliId].msgId; + QByteArray clientInstanceId = pCacheItem->clientMap[cliId].instanceId; + QByteArray clientFrom = pCacheItem->clientMap[cliId].from; + + log_debug("[SUBSCRIBE] Sending Subscription update to client id=%s, msgId=%s, instanceId=%s", + cliId.data(), qPrintable(clientMsgId), clientInstanceId.data()); + + writeToClient__(CacheResponse, p, cliId, clientFrom, clientInstanceId); + + ++it; + } + else + { + it = pCacheItem->clientMap.erase(it); // erase returns the next valid iterator + } + } + } + } + } + } + } + + QTimer::singleShot(1 * 1000, [=]() { + scan_subscribe_update(); + }); + } + + void unregister_client(const QByteArray& clientId) + { + if (gHttpClientMap.contains(clientId)) + { + gHttpClientMap.remove(clientId); + log_debug("[HTTP] Deleted http client=%s", clientId.data()); + } + else + { + // delete client from gWsClientMap + gWsClientMap.remove(clientId); + log_debug("[WS] Deleted ws client=%s", clientId.data()); + } + } + + void register_http_client(QByteArray packetId, QByteArray from, QString urlPath, int creditSize) + { + if (gHttpClientMap.contains(packetId)) + { + log_debug("[HTTP] already exists http client id=%s", packetId.data()); + return; + } + + struct ClientItem clientItem; + clientItem.lastRequestSeq = 0; + clientItem.lastResponseSeq = -1; + clientItem.lastRequestTime = QDateTime::currentMSecsSinceEpoch(); + clientItem.lastResponseTime = QDateTime::currentMSecsSinceEpoch(); + clientItem.from = from; + clientItem.creditSize = creditSize > 8192 ? creditSize : 8192; + clientItem.urlPath = urlPath; + gHttpClientMap[packetId] = clientItem; + log_debug("[HTTP] added http client id=%s", packetId.data()); + + return; + } + + void register_ws_client(QByteArray packetId, QByteArray from, QString urlPath, int creditSize) + { + if (gWsClientMap.contains(packetId)) + { + log_debug("[WS] already exists ws client id=%s", packetId.data()); + return; + } + + struct ClientItem clientItem; + clientItem.lastRequestSeq = 0; + clientItem.lastResponseSeq = -1; + clientItem.lastRequestTime = QDateTime::currentMSecsSinceEpoch(); + clientItem.lastResponseTime = QDateTime::currentMSecsSinceEpoch(); + clientItem.from = from; + clientItem.creditSize = creditSize > 8192 ? creditSize : 8192; + clientItem.urlPath = urlPath; + gWsClientMap[packetId] = clientItem; + log_debug("[WS] added ws client id=%s", packetId.data()); + + QTimer::singleShot(PING_INTERVAL * 1000, [=]() { + timer_send_ping_to_client(packetId); + }); + + return; + } + + void register_http_cache_item( + const ZhttpRequestPacket &clientPacket, + QByteArray clientId, + const PacketMsg &packetMsg, + int backendNo) + { + // create new cache item + struct CacheItem cacheItem; + cacheItem.newMsgId = -1; + cacheItem.refreshFlag = 0x00; + if (is_never_timeout_method(packetMsg.method, packetMsg.params)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_NEVER_TIMEOUT; + log_debug("[HTTP] added refresh never timeout method"); + } + if (gRefreshShorterMethodList.contains(packetMsg.method, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_SHORTER_TIMEOUT; + log_debug("[HTTP] added refresh shorter method"); + } + if (gRefreshLongerMethodList.contains(packetMsg.method, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_LONGER_TIMEOUT; + log_debug("[HTTP] added refresh longer method"); + } + if (gRefreshUneraseMethodList.contains(packetMsg.method, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_UNERASE; + log_debug("[HTTP] added refresh unerase method"); + } + if (gRefreshExcludeMethodList.contains(packetMsg.method, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_EXCLUDE; + log_debug("[HTTP] added refresh exclude method"); + } + if (gRefreshPassthroughMethodList.contains(packetMsg.method, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_PASSTHROUGH; + log_debug("[HTTP] added refresh passthrough method"); + } + if (gNullResponseMethodList.contains(packetMsg.method, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= ACCEPT_NULL_RESPONSE; + log_debug("[HTTP] added null response method"); + } + cacheItem.lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + cacheItem.lastRefreshCount = 0; + cacheItem.lastAccessTime = QDateTime::currentMSecsSinceEpoch(); + cacheItem.lastRequestTime = QDateTime::currentMSecsSinceEpoch(); + cacheItem.cachedFlag = false; + + cacheItem.methodName = packetMsg.method; + + // check cache/subscribe method + if (is_cache_method(packetMsg.method)) + { + cacheItem.methodType = CACHE_METHOD; + } + else if (is_subscribe_method(packetMsg.method)) + { + cacheItem.methodType = SUBSCRIBE_METHOD; + } + + // save the request packet with new id + cacheItem.requestPacket = clientPacket; + replace_id_field(cacheItem.requestPacket.body, packetMsg.id, QString("__MSGID__")); + cacheItem.clientMap[clientId].msgId = packetMsg.id; + cacheItem.clientMap[clientId].from = clientPacket.from; + cacheItem.clientMap[clientId].instanceId = instanceId; + cacheItem.proto = Scheme::http; + cacheItem.retryCount = 0; + cacheItem.httpBackendNo = backendNo; + + create_cache_item(packetMsg.paramsHash, cacheItem); + + log_debug("[HTTP] Registered New Cache Item for id=%s method=\"%s\" backend=%d", qPrintable(packetMsg.id), qPrintable(packetMsg.method), backendNo); + } + + int register_ws_cache_item( + const ZhttpRequestPacket &clientPacket, + QByteArray clientId, + QString orgMsgId, + QString methodName, + QString msgParams, + const QByteArray &methodNameParamsHashVal) + { + // create new cache item + struct CacheItem cacheItem; + + int ccIndex = get_main_cc_index(instanceId); + if (ccIndex < 0) + return -1; + cacheItem.newMsgId = gWsCacheClientList[ccIndex].msgIdCount; + cacheItem.refreshFlag = 0x00; + if (is_never_timeout_method(methodName, msgParams)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_NEVER_TIMEOUT; + log_debug("[WS] added refresh never timeout method"); + } + if (gRefreshShorterMethodList.contains(methodName, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_SHORTER_TIMEOUT; + log_debug("[WS] added refresh shorter method"); + } + if (gRefreshLongerMethodList.contains(methodName, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_LONGER_TIMEOUT; + log_debug("[WS] added refresh longer method"); + } + if (gRefreshUneraseMethodList.contains(methodName, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_UNERASE; + log_debug("[WS] added refresh unerase method"); + } + if (gRefreshExcludeMethodList.contains(methodName, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_EXCLUDE; + log_debug("[WS] added refresh exclude method"); + } + if (gRefreshPassthroughMethodList.contains(methodName, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= AUTO_REFRESH_PASSTHROUGH; + log_debug("[WS] added refresh passthrough method"); + } + if (gNullResponseMethodList.contains(methodName, Qt::CaseInsensitive)) + { + cacheItem.refreshFlag |= ACCEPT_NULL_RESPONSE; + log_debug("[WS] added null response method"); + } + cacheItem.lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + cacheItem.lastRefreshCount = 0; + cacheItem.lastAccessTime = QDateTime::currentMSecsSinceEpoch(); + cacheItem.cachedFlag = false; + + // save the request packet with new id + cacheItem.requestPacket = clientPacket; + replace_id_field(cacheItem.requestPacket.body, orgMsgId, QString("__MSGID__")); + cacheItem.clientMap[clientId].msgId = orgMsgId; + cacheItem.clientMap[clientId].from = clientPacket.from; + cacheItem.clientMap[clientId].instanceId = instanceId; + cacheItem.proto = Scheme::websocket; + cacheItem.retryCount = 0; + cacheItem.cacheClientId = gWsCacheClientList[ccIndex].clientId; + cacheItem.methodName = methodName; + cacheItem.subscriptionUpdateCount = 0; + + // check cache/subscribe method + if (is_cache_method(methodName)) + { + cacheItem.methodType = CACHE_METHOD; + } + else if (is_subscribe_method(methodName)) + { + cacheItem.methodType = SUBSCRIBE_METHOD; + } + + create_cache_item(methodNameParamsHashVal, cacheItem); + + return ccIndex; + } + + int process_http_request(QByteArray id, const ZhttpRequestPacket &p, const QString &urlPath) + { + QByteArray packetId = id; + + // parse json body + PacketMsg packetMsg; + if (parse_packet_msg(Scheme::http, p, packetMsg, instanceId) < 0) + return -1; + + // get method string + if (packetMsg.id.isEmpty() || packetMsg.method.isEmpty()) + { + log_debug("[HTTP] failed to get gMsgIdAttrName and gMsgMethodAttrName"); + return -1; + } + log_debug("[HTTP] new req msgId=%s method=%s msgParams=%s", + qPrintable(packetMsg.id), qPrintable(packetMsg.method), qPrintable(packetMsg.params)); + + // update the counter for prometheus + count_requests(packetMsg.method); + + if (is_cache_method(packetMsg.method)) + { + CacheItem *pCacheItem = load_cache_item(packetMsg.paramsHash); + if (pCacheItem != NULL) + { + pCacheItem->lastAccessTime = QDateTime::currentMSecsSinceEpoch(); + + // prometheus staus + update_prometheus_hit_count(*pCacheItem); + + // send credit response to client + send_response_to_client(ZhttpResponsePacket::Credit, packetId, p.from, static_cast(p.body.size())); + + if (pCacheItem->cachedFlag == true) + { + writeToClient_(packetMsg.paramsHash, packetId, packetMsg.id, p.from, instanceId); + log_debug("[HTTP] Replied with Cache content for method \"%s\"", qPrintable(packetMsg.method)); + unregister_client(packetId); + count_responses("HTTP"); + } + else + { + log_debug("[HTTP] Already cache registered, but not added content \"%s\"", qPrintable(packetMsg.method)); + // add client to list + pCacheItem->clientMap[packetId].msgId = packetMsg.id; + pCacheItem->clientMap[packetId].from = p.from; + pCacheItem->clientMap[packetId].instanceId = instanceId; + log_debug("[HTTP] Adding new client id msgId=%s clientId=%s", qPrintable(packetMsg.id), packetId.data()); + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + } + + return 0; + } + else + { + log_debug("[HTTP] not found in cache"); + } + + int backendNo = -1; + for (int i = 0; i < gHttpBackendUrlList.count(); i++) + { + if (urlPath == gHttpBackendUrlList[i]) + { + backendNo = i; + break; + } + } + + // Register new cache item + register_http_cache_item(p, packetId, packetMsg, backendNo); + + // register cache refresh + register_cache_refresh(packetMsg.paramsHash, urlPath); + } + + return -1; + } + + int process_http_response(const ZhttpResponsePacket &responsePacket, const QByteArray &instanceAddress) + { + QVariant vpacket1 = responsePacket.toVariant(); + QByteArray responseBuf1 = instanceAddress + " T" + TnetString::fromVariant(vpacket1); + + ZhttpResponsePacket p = responsePacket; + QByteArray packetId = p.ids[0].id; + QByteArray from = p.from; + + // check multi-part response + int ret = check_multi_packets_for_http_response(p); + if (ret < 0) + return 0; + + QVariant vpacket = p.toVariant(); + QByteArray responseBuf = instanceAddress + " T" + TnetString::fromVariant(vpacket); + + bool bodyParseSucceed = true; + + // parse json body + PacketMsg packetMsg; + if (parse_packet_msg(Scheme::http, p, packetMsg, instanceId) < 0) + bodyParseSucceed = false; + + CacheItem *pCacheItem = NULL; + QByteArray itemId = QByteArray(); + if (bodyParseSucceed == true) + { + // convert to QByteArray + QString tmpStr = packetMsg.id; + QByteArray msgIdByte = QByteArray::fromHex(qPrintable(tmpStr.remove('\"'))); + + pCacheItem = load_cache_item(msgIdByte); + if (pCacheItem != NULL) + { + itemId = msgIdByte; + } + } + + if (itemId.isEmpty()) + { + foreach(QByteArray _itemId, get_cache_item_ids()) + { + pCacheItem = load_cache_item(_itemId); + if (pCacheItem == NULL) + { + log_debug("[HTTP] not found cache item %s", _itemId.toHex().data()); + continue; + } + if ((pCacheItem->proto == Scheme::http) && + (pCacheItem->requestPacket.ids[0].id == packetId) && + (pCacheItem->cachedFlag == false) + ) + { + itemId = _itemId; + break; + } + } + } + + if (itemId.isEmpty() || pCacheItem->proto != Scheme::http) + { + log_debug("[HTTP] could not find the cache item %s", itemId.constData()); + return -1; + } + + // if invalid response? + if ((bodyParseSucceed == false || (pCacheItem->cachedFlag == false && packetMsg.isResultNull == true)) && + !(pCacheItem->refreshFlag & ACCEPT_NULL_RESPONSE) && + pCacheItem->retryCount < (gHttpBackendUrlList.count()+1)) + { + // prometheus status + if (pCacheItem->httpBackendNo >= 0) + { + QString urlPath = gHttpBackendUrlList[pCacheItem->httpBackendNo]; + if (httpCacheClientInvalidResponseCountMap.contains(urlPath)) + httpCacheClientInvalidResponseCountMap[urlPath]++; + } + + log_debug("[HTTP] get NULL response, retrying %d", pCacheItem->retryCount); + pCacheItem->lastAccessTime = QDateTime::currentMSecsSinceEpoch(); + + //store_cache_item_field(msgIdByte, "lastAccessTime", pCacheItem->lastAccessTime); + return -1; + } + + // send response to clients. + pCacheItem->newMsgId = 0; + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + pCacheItem->cachedFlag = true; + log_debug("[HTTP] Added/Updated Cache content for method=%s", qPrintable(pCacheItem->methodName)); + + // store response body + store_cache_response_buffer(itemId, responseBuf, packetMsg.id, 0); + + foreach(QByteArray cliId, pCacheItem->clientMap.keys()) + { + if (gHttpClientMap.contains(cliId) && cliId != packetId) + { + QString msgId = pCacheItem->clientMap[cliId].msgId; + QByteArray orgInstanceId = pCacheItem->clientMap[cliId].instanceId; + writeToClient_(itemId, cliId, msgId, instanceAddress, orgInstanceId); + + log_debug("[HTTP] Sent Cache content to client id=%s", cliId.data()); + unregister_client(cliId); + count_responses("HTTP"); + } + } + pCacheItem->clientMap.clear(); + + return 0; + } + + int process_ws_cacheclient_response(const ZhttpResponsePacket &response, int cacheClientNumber, const QByteArray &instanceAddress) + { + ZhttpResponsePacket p = response; + + // check multi-part response + int ret = check_multi_packets_for_ws_response(p); + if (ret < 0) + return -1; + + QByteArray packetId = p.ids[0].id; + + QVariant vpacket = p.toVariant(); + QByteArray packetBuf = TnetString::fromVariant(vpacket); + QByteArray responseBuf = instanceAddress + " T" + packetBuf; + + if(log_outputLevel() >= LOG_LEVEL_DEBUG) + LogUtil::logVariantWithContent(LOG_LEVEL_DEBUG, vpacket, "body", "%s server: OUT %s", "[CacheClient]", instanceAddress.data()); + + // parse json body + PacketMsg packetMsg; + if (parse_packet_msg(Scheme::websocket, p, packetMsg, instanceId) < 0) + return -1; + + // id + int msgIdValue = is_convertible_to_int(packetMsg.id) ? packetMsg.id.toInt() : -1; + + // result + QString msgResultStr = packetMsg.result; + + // if it is curie response without change, ignore + QString methodName = packetMsg.method; + + if (!packetMsg.subscription.isEmpty()) + { + QString subscriptionStr = packetMsg.subscription; + + foreach(QByteArray itemId, get_cache_item_ids()) + { + CacheItem* pCacheItem = load_cache_item(itemId); + if (pCacheItem == NULL) + { + log_debug("[WS] not found cache item %s", itemId.toHex().data()); + continue; + } + if (pCacheItem->subscriptionStr == subscriptionStr) + { + if (pCacheItem->cachedFlag == false) + { + pCacheItem->cachedFlag = true; + pCacheItem->subscriptionUpdateCount = 0; + + // update block and changes + if (!packetMsg.resultBlock.isEmpty()) + { + QString msgBlockStr = packetMsg.resultBlock.toLower(); + pCacheItem->blockStr = msgBlockStr; + } + + if (!packetMsg.resultChanges.isEmpty()) + { + QString msgChangesStr = packetMsg.resultChanges.toLower(); + + QStringList changesList = msgChangesStr.split("/"); + for ( const auto& changes : changesList ) + { + QStringList changeList = changes.split("+"); + if (changeList.size() != 2) + { + log_debug("[WS] Invalid change list"); + continue; + } + pCacheItem->changesMap[changeList[0]] = changeList[1]; + } + } + + // store response body + QByteArray subscriptionKey = itemId + "-sub"; + store_cache_response_buffer(subscriptionKey, responseBuf, QString(""), 0); + + log_debug("[WS] Added Subscription content for subscription method id=%d subscription=%s", + pCacheItem->newMsgId, qPrintable(subscriptionStr)); + // send update subscribe to all clients + QHash::iterator it = pCacheItem->clientMap.begin(); + while (it != pCacheItem->clientMap.end()) + { + QByteArray cliId = it.key(); + if (gWsClientMap.contains(cliId)) + { + QString clientMsgId = pCacheItem->clientMap[cliId].msgId; + QByteArray clientInstanceId = pCacheItem->clientMap[cliId].instanceId; + + log_debug("[WS] Sending Subscription content to client id=%s, msgId=%s, instanceId=%s", + cliId.data(), qPrintable(clientMsgId), clientInstanceId.data()); + + writeToClient_(itemId, cliId, clientMsgId, instanceAddress, clientInstanceId); + writeToClient_(subscriptionKey, cliId, clientMsgId, instanceAddress, clientInstanceId); + + ++it; + } + else + { + it = pCacheItem->clientMap.erase(it); // erase returns the next valid iterator + } + } + } + else + { + QByteArray subscriptionKey = itemId + "-sub"; + if (!packetMsg.resultBlock.isEmpty() || !packetMsg.resultChanges.isEmpty()) + { + QByteArray responseBuf_ = load_cache_response_buffer(instanceAddress, subscriptionKey, packetId, 0, QString("__ID__"), "__FROM__", 0); + + int diffLen = 0; + + // update block and changes + if (!packetMsg.resultBlock.isEmpty()) + { + QString newBlockStr = packetMsg.resultBlock.toLower(); + QString oldBlckStr = pCacheItem->blockStr; + + QByteArray oldPatternStr = "\"block\":"; + oldPatternStr += "\""; + oldPatternStr += oldBlckStr.toUtf8(); + oldPatternStr += "\""; + QByteArray newPatternStr = "\"block\":"; + newPatternStr += "\""; + newPatternStr += newBlockStr.toUtf8(); + newPatternStr += "\""; + + qsizetype idxStart = responseBuf_.indexOf(oldPatternStr); + if (idxStart >= 0) + { + log_debug("[PPP-1] %s,%s", oldPatternStr.constData(), newPatternStr.constData()); + responseBuf_.replace(idxStart, oldPatternStr.length(), newPatternStr); + diffLen += newPatternStr.length() - oldPatternStr.length(); + pCacheItem->blockStr = newBlockStr; + } + } + ///* + if (!packetMsg.resultChanges.isEmpty()) + { + QString msgChangesStr = packetMsg.resultChanges.toLower(); + + QStringList changesList = msgChangesStr.split("/"); + for ( const auto& changes : changesList ) + { + QStringList changeList = changes.split("+"); + if (changeList.size() != 2) + { + log_debug("[WS] Invalid change list"); + continue; + } + + QString changesKey = changeList[0]; + QString oldVal = pCacheItem->changesMap[changeList[0]]; + QString newVal = changeList[1]; + + QByteArray oldPatternStr = "[\""; + oldPatternStr += changesKey.toUtf8(); + oldPatternStr += (oldVal != "null") ? "\",\"" : "\","; + oldPatternStr += oldVal.toUtf8(); + oldPatternStr += (oldVal != "null") ? "\"]" : "]"; + QByteArray newPatternStr = "[\""; + newPatternStr += changesKey.toUtf8(); + newPatternStr += (newVal != "null") ? "\",\"" : "\","; + newPatternStr += newVal.toUtf8(); + newPatternStr += (newVal != "null") ? "\"]" : "]"; + + qsizetype idxStart = responseBuf_.indexOf(oldPatternStr); + if (idxStart >= 0) + { + log_debug("[PPP-2] %s,%s", oldPatternStr.constData(), newPatternStr.constData()); + responseBuf_.replace(idxStart, oldPatternStr.length(), newPatternStr); + diffLen += newPatternStr.length() - oldPatternStr.length(); + pCacheItem->changesMap[changesKey] = newVal; + } + } + } + //*/ + + /* + QByteArray patternStr = "\"block\":\""; + qsizetype idxStart = responseBuf_.indexOf(patternStr); + if (idxStart >= 0) + { + qsizetype idxEnd = responseBuf_.indexOf("\"", idxStart+9); + responseBuf_.replace(idxStart+9, idxEnd-(idxStart+9), QByteArray(qPrintable(msgBlockStr))); + } + else + { + log_debug("[WS] not found block in subscription cached response"); + } + + QString msgChangesStr = packetMsg.resultChanges.toLower(); + QStringList changesList = msgChangesStr.split("/"); + for ( const auto& changes : changesList ) + { + QStringList changeList = changes.split("+"); + if (changeList.size() != 2) + { + log_debug("[WS] Invalid change list"); + continue; + } + + log_debug("[QQQ] %s,%s", qPrintable(changeList[0]), qPrintable(changeList[1])); + + QString patternStr(qPrintable("[\"" + changeList[0] + "\"")); + QString newPattern = "[\""; + newPattern += changeList[0]; + newPattern += "\",\""; + newPattern += changeList[1]; + newPattern += "\"]"; + + qsizetype idxStart = 0; + qsizetype idxEnd = 0; + while (1) + { + idxStart = responseBuf_.indexOf(patternStr, idxEnd); + if (idxStart < 0) + break; + + idxEnd = responseBuf_.indexOf("]", idxStart+changeList[0].length()); + if (idxEnd > idxStart) + { + responseBuf_.replace(idxStart, idxEnd-idxStart+1, qPrintable(newPattern)); + diffLen += newPattern.length() - (idxEnd-idxStart+1); + } + else + { + log_debug("[WS] not found change param in subscription cached response"); + break; + } + } + } + */ + + // store response body + store_cache_response_buffer(subscriptionKey, responseBuf_, QString(""), diffLen); + } + else + { + // store response body + store_cache_response_buffer(subscriptionKey, responseBuf, QString(""), 0); + } + + // update subscription last update time + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + + // save update packet into redis + if (gRedisEnable == true && gReplicaFlag == false) + { + QByteArray updateKey = itemId + "-update"; + redis_store_cache_response(updateKey, packetBuf); + + QByteArray updateCountKey = itemId + "-updateCount"; + pCacheItem->subscriptionUpdateCount++; + QByteArray countBytes = QByteArray::number(pCacheItem->subscriptionUpdateCount); + redis_store_cache_response(updateCountKey, countBytes); + } + + // send update subscribe to all clients + QHash::iterator it = pCacheItem->clientMap.begin(); + while (it != pCacheItem->clientMap.end()) + { + QByteArray cliId = it.key(); + if (gWsClientMap.contains(cliId)) + { + QString clientMsgId = pCacheItem->clientMap[cliId].msgId; + QByteArray clientInstanceId = pCacheItem->clientMap[cliId].instanceId; + + log_debug("[WS] Sending Subscription update to client id=%s, msgId=%s, instanceId=%s", + cliId.data(), qPrintable(clientMsgId), clientInstanceId.data()); + + writeToClient__(CacheResponse, p, cliId, instanceAddress, clientInstanceId); + + ++it; + } + else + { + it = pCacheItem->clientMap.erase(it); // erase returns the next valid iterator + } + } + } + + return -1; + } + } + + // create new subscription item + struct CacheItem cacheItem; + cacheItem.newMsgId = -1; + cacheItem.lastRequestTime = QDateTime::currentMSecsSinceEpoch(); + cacheItem.lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + cacheItem.cachedFlag = false; + cacheItem.methodType = CacheMethodType::SUBSCRIBE_METHOD; + cacheItem.subscriptionStr = subscriptionStr; + cacheItem.subscriptionUpdateCount = 0; + cacheItem.cacheClientId = gWsCacheClientList[cacheClientNumber].clientId; + + // update block and changes + if (!packetMsg.resultBlock.isEmpty()) + { + QString msgBlockStr = packetMsg.resultBlock.toLower(); + cacheItem.blockStr = msgBlockStr; + } + + if (!packetMsg.resultChanges.isEmpty()) + { + QString msgChangesStr = packetMsg.resultChanges.toLower(); + + QStringList changesList = msgChangesStr.split("/"); + for ( const auto& changes : changesList ) + { + QStringList changeList = changes.split("+"); + if (changeList.size() != 2) + { + log_debug("[WS] Invalid change list"); + continue; + } + cacheItem.changesMap[changeList[0]] = changeList[1]; + } + } + + // store response body + store_cache_response_buffer(subscriptionStr.toUtf8(), responseBuf, QString(""), 0); + + QByteArray subscriptionBytes = subscriptionStr.toUtf8(); + create_cache_item(subscriptionBytes, cacheItem); + log_debug("[WS] Registered Subscription for \"%s\"", qPrintable(subscriptionStr)); + + // make invalild + return -1; + } + + if(msgIdValue < 0) + { + // make invalild + log_debug("[WS] detected response without id"); + return -1; + } + + foreach(QByteArray itemId, get_cache_item_ids()) + { + CacheItem* pCacheItem = load_cache_item(itemId); + if (pCacheItem == NULL) + { + log_debug("[SCAN] not found cache item %s", itemId.toHex().data()); + continue; + } + if ((pCacheItem->proto == Scheme::websocket) && + (pCacheItem->newMsgId == msgIdValue) && + (pCacheItem->cacheClientId == packetId)) + { + if (pCacheItem->methodType == CacheMethodType::CACHE_METHOD) + { + log_debug("[WS] Adding Cache content for method name=%s", qPrintable(pCacheItem->methodName)); + + if (pCacheItem->cachedFlag == false && + !(pCacheItem->refreshFlag & ACCEPT_NULL_RESPONSE) && + packetMsg.isResultNull == true && + pCacheItem->retryCount < gWsBackendUrlList.count()) + { + log_debug("[WS] get NULL response, retrying %d", pCacheItem->retryCount); + pCacheItem->lastAccessTime = QDateTime::currentMSecsSinceEpoch(); + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + + int ccIndex = get_cc_index_from_clientId(pCacheItem->cacheClientId); + // prometheus status + if (ccIndex >= 0) + { + QString urlPath = gWsBackendUrlList[ccIndex]; + if (wsCacheClientInvalidResponseCountMap.contains(urlPath)) + wsCacheClientInvalidResponseCountMap[urlPath]++; + } + + return 0; + } + + pCacheItem->newMsgId = msgIdValue; + pCacheItem->cachedFlag = true; + + // store response body + store_cache_response_buffer(itemId, responseBuf, packetMsg.id, 0); + + // send response to all clients + QString urlPath = ""; + foreach(QByteArray cliId, pCacheItem->clientMap.keys()) + { + if (gWsClientMap.contains(cliId)) + { + QString clientMsgId = pCacheItem->clientMap[cliId].msgId; + QByteArray clientInstanceId = pCacheItem->clientMap[cliId].instanceId; + + if (urlPath.isEmpty()) + urlPath = gWsClientMap[cliId].urlPath; + log_debug("[WS] Sending Cache content to client id=%s", cliId.data()); + writeToClient_(itemId, cliId, clientMsgId, instanceAddress, clientInstanceId); + } + } + pCacheItem->clientMap.clear(); + + // delete cache item once sent response if cache-less one connection is enabled. + if (pCacheItem->refreshFlag & AUTO_REFRESH_PASSTHROUGH) + { + log_debug("[WS] Delete cache item because no auto-refresh"); + remove_cache_item(itemId); + } + } + else if (pCacheItem->methodType == CacheMethodType::SUBSCRIBE_METHOD) + { + log_debug("[WS] Adding Subscribe content for method name=%s", qPrintable(pCacheItem->methodName)); + + // result + if(msgResultStr.isNull()) + { + return -1; + } + pCacheItem->newMsgId = msgIdValue; + if ((msgResultStr.compare("true", Qt::CaseInsensitive) != 0) && (msgResultStr.compare("false", Qt::CaseInsensitive) != 0)) + { + pCacheItem->subscriptionStr = msgResultStr; + } + else + { + return -1; + } + + log_debug("[WS] Registered Subscription result for \"%s\"", qPrintable(msgResultStr)); + + // update subscription last update time + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + + // store response body + store_cache_response_buffer(itemId, responseBuf, packetMsg.id, 0); + + // Search temp teim in SubscriptionItemMap + QByteArray resultBytes = msgResultStr.toUtf8(); + CacheItem* pResultCacheItem = load_cache_item(resultBytes); + if (pResultCacheItem != NULL) + { + if (pResultCacheItem->newMsgId == -1) + { + pCacheItem->cachedFlag = true; + pCacheItem->subscriptionUpdateCount = 0; + remove_cache_item(resultBytes); + log_debug("[WS] Added Subscription content for subscription method id=%d result=%s", msgIdValue, qPrintable(msgResultStr)); + } + } + + if (pCacheItem->cachedFlag == true) + { + QByteArray subscriptionKey = itemId + "-sub"; + // send update subscribe to all clients + QHash::iterator it = pCacheItem->clientMap.begin(); + while (it != pCacheItem->clientMap.end()) + { + QByteArray cliId = it.key(); + if (gWsClientMap.contains(cliId)) + { + QString clientMsgId = pCacheItem->clientMap[cliId].msgId; + QByteArray clientInstanceId = pCacheItem->clientMap[cliId].instanceId; + + log_debug("[WS] Sending Subscription content to client id=%s, msgId=%s, instanceId=%s", + cliId.data(), qPrintable(clientMsgId), clientInstanceId.data()); + + writeToClient_(itemId, cliId, clientMsgId, instanceAddress, clientInstanceId); + writeToClient_(subscriptionKey, cliId, clientMsgId, instanceAddress, clientInstanceId); + + ++it; + } + else + { + it = pCacheItem->clientMap.erase(it); // erase returns the next valid iterator + } + } + } + } + return -1; + } + } + + return 0; + } + + int send_ws_request_over_cacheclient(const ZhttpRequestPacket &packet, QString orgMsgId, int ccIndex) + { + if (ccIndex < 0 || gWsCacheClientList[ccIndex].initFlag == false) + { + log_debug("[WS] Invalid cache client %d", ccIndex); + return -1; + } + + // Create new packet by cache client + ZhttpRequestPacket p = packet; + ClientItem *cacheClient = &gWsCacheClientList[ccIndex]; + + ZhttpRequestPacket::Id tempId; + tempId.id = cacheClient->clientId; // id + tempId.seq = update_request_seq(cacheClient->clientId); + p.ids.clear(); + p.ids += tempId; + + int msgId = -1; + if (!orgMsgId.isEmpty()) + { + cacheClient->msgIdCount += 1; + msgId = cacheClient->msgIdCount; + replace_id_field(p.body, orgMsgId, msgId); + } + + // log + if(log_outputLevel() >= LOG_LEVEL_DEBUG) + { + QString vrespStr = TnetString::variantToString(p.toVariant(), -1); + QString logStr; + if (vrespStr.length() > DEBUG_LOG_MAX_LENGTH) + { + logStr = vrespStr.leftRef(DEBUG_LOG_MAX_LENGTH/2) + "........" + vrespStr.rightRef(DEBUG_LOG_MAX_LENGTH/2); + } + else + { + logStr = vrespStr; + } + log_debug("[WS] send_ws_request_over_cacheclient: %s", qPrintable(logStr)); } - QPointer self = this; + std::weak_ptr self = q->d; foreach(const ZhttpRequestPacket::Id &id, p.ids) { // is this for a websocket? - ZWebSocket *sock = serverSocksByRid.value(ZWebSocket::Rid(p.from, id.id)); + ZWebSocket *sock = serverSocksByRid.value(ZWebSocket::Rid(cacheClient->from, id.id)); if(sock) { sock->handle(id.id, id.seq, p); - if(!self) - return; - - continue; - } - - // is this for an http request? - ZhttpRequest *req = serverReqsByRid.value(ZhttpRequest::Rid(p.from, id.id)); - if(req) - { - req->handle(id.id, id.seq, p); - if(!self) - return; + if(self.expired()) + return -1; continue; } - - log_debug("zhttp/zws server: received message for unknown request id, skipping"); } + return msgId; } - void refresh_timeout() + int send_unsubscribe_request_over_cacheclient() { - QHash > clientSessionsBySender[2]; // index corresponds to type - QHash > serverSessionsBySender[2]; // index corresponds to type + int itemCount = gUnsubscribeRequestMap[instanceId].count(); + if (itemCount > 0) + { + UnsubscribeRequestItem reqItem = gUnsubscribeRequestMap[instanceId][0]; + gUnsubscribeRequestMap[instanceId].removeAt(0); - // process the current bucket - const QSet &bucket = sessionRefreshBuckets[currentSessionRefreshBucket]; - foreach(KeepAliveRegistration *r, bucket) - { - QPair rid; - bool isServer; - if(r->type == HttpSession) - { - rid = r->p.req->rid(); - isServer = r->p.req->isServer(); - } - else // WebSocketSession - { - rid = r->p.sock->rid(); - isServer = r->p.sock->isServer(); - } + // Create new packet by cache client + ZhttpRequestPacket p; + ZhttpRequestPacket::Id tempId; - QByteArray sender; - if(isServer) - { - sender = rid.first; - } - else + int ccIndex = get_cc_index_from_clientId(reqItem.cacheClientId); + + if (ccIndex < 0 || gWsCacheClientList[ccIndex].initFlag == false) { - if(r->type == HttpSession) - sender = r->p.req->toAddress(); - else // WebSocketSession - sender = r->p.sock->toAddress(); + log_debug("[WS] Invalid cache client %d", ccIndex); + return -1; } - assert(!sender.isEmpty()); + ClientItem *cacheClient = &gWsCacheClientList[ccIndex]; - QHash > &sessionsBySender = (isServer ? serverSessionsBySender[r->type - 1] : clientSessionsBySender[r->type - 1]); + tempId.id = gWsCacheClientList[ccIndex].clientId; // id + tempId.seq = update_request_seq(cacheClient->clientId); + p.ids.append(tempId); - if(!sessionsBySender.contains(sender)) - sessionsBySender.insert(sender, QList()); + p.type = ZhttpRequestPacket::Data; + p.from = reqItem.from; - QList &sessions = sessionsBySender[sender]; - sessions += r; + char bodyStr[1024]; + gWsCacheClientList[ccIndex].msgIdCount++; + int msgId = gWsCacheClientList[ccIndex].msgIdCount; + QString methodName = reqItem.unsubscribeMethodName; + qsnprintf(bodyStr, 1024, "{\"id\":%d,\"jsonrpc\":\"2.0\",\"method\":\"%s\",\"params\":[\"%s\"]}", + msgId, qPrintable(methodName), qPrintable(reqItem.subscriptionStr)); + p.body = QByteArray(bodyStr); - // if we're at max, send out now - if(sessions.count() >= ZHTTP_IDS_MAX) + log_debug("[WS] send_unsubscribeRequest: %s", qPrintable(TnetString::variantToString(p.toVariant(), -1))); + + std::weak_ptr self = q->d; + + foreach(const ZhttpRequestPacket::Id &id, p.ids) { - if(isServer) + // is this for a websocket? + ZWebSocket *sock = serverSocksByRid.value(ZWebSocket::Rid(cacheClient->from, id.id)); + if(sock) { - QList ids; - foreach(KeepAliveRegistration *i, sessions) - { - assert(i->type == r->type); - if(r->type == HttpSession) - ids += ZhttpResponsePacket::Id(i->p.req->rid().second, i->p.req->outSeqInc()); - else // WebSocketSession - ids += ZhttpResponsePacket::Id(i->p.sock->rid().second, i->p.sock->outSeqInc()); - } + sock->handle(id.id, id.seq, p); + if(self.expired()) + return -1; - writeKeepAlive(r->type, ids, sender); + continue; } - else - { - QList ids; - foreach(KeepAliveRegistration *i, sessions) - { - assert(i->type == r->type); - if(r->type == HttpSession) - ids += ZhttpRequestPacket::Id(i->p.req->rid().second, i->p.req->outSeqInc()); - else // WebSocketSession - ids += ZhttpRequestPacket::Id(i->p.sock->rid().second, i->p.sock->outSeqInc()); - } + } + } - writeKeepAlive(r->type, ids, sender); - } + return 0; + } - sessions.clear(); - sessionsBySender.remove(sender); - } + void delete_old_clients() + { + int itemCount = gDeleteClientList.count(); + if (itemCount > 0) + { + QByteArray clientId = gDeleteClientList[0]; + gDeleteClientList.removeAt(0); + + unregister_client(clientId); } + } - // send last packets - for(int n = 0; n < 2; ++n) + int process_ws_stream_request(const QByteArray packetId, ZhttpRequestPacket &p) + { + int ret = check_multi_packets_for_ws_request(p); + if (ret < 0) + return -1; + + // parse json body + PacketMsg packetMsg; + if (parse_packet_msg(Scheme::websocket, p, packetMsg, instanceId) < 0) + return -1; + + // read msgIdStr (id) and methodName (method) + QString msgIdStr = packetMsg.id; + QString methodName = packetMsg.method; + QString msgParams = packetMsg.params; + if (msgIdStr.isEmpty() || methodName.isEmpty()) { - SessionType type = (SessionType)(n + 1); + log_debug("[WS] failed to get gMsgIdAttrName and gMsgMethodAttrName"); + return 0; + } + + // get method string + log_debug("[WS] Cache entry msgId=\"%s\" method=\"%s\"", qPrintable(msgIdStr), qPrintable(methodName)); + + // update the counter for prometheus + count_requests(methodName); + + // Params hash val + QByteArray paramsHash = packetMsg.paramsHash; + if (is_cache_method(methodName) || is_subscribe_method(methodName)) + { + CacheItem* pCacheItem = load_cache_item(paramsHash, methodName); + if (pCacheItem != NULL) { - QHashIterator > sit(clientSessionsBySender[n]); - while(sit.hasNext()) - { - sit.next(); - const QByteArray &sender = sit.key(); - const QList &sessions = sit.value(); + pCacheItem->lastAccessTime = QDateTime::currentMSecsSinceEpoch(); - if(!sessions.isEmpty()) + // prometheus staus + update_prometheus_hit_count(*pCacheItem); + + if (pCacheItem->cachedFlag == true) + { + int ccIndex = get_cc_index_from_clientId(pCacheItem->cacheClientId); + if (ccIndex < 0 || gWsCacheClientList[ccIndex].initFlag == false) { - QList ids; - foreach(KeepAliveRegistration *i, sessions) + ccIndex = get_main_cc_index(instanceId); + if (ccIndex < 0) { - assert(i->type == type); - if(type == HttpSession) - ids += ZhttpRequestPacket::Id(i->p.req->rid().second, i->p.req->outSeqInc()); - else // WebSocketSession - ids += ZhttpRequestPacket::Id(i->p.sock->rid().second, i->p.sock->outSeqInc()); + log_warning("[WS] not initialized cache client, ignore"); + return 0; } + } - writeKeepAlive(type, ids, sender); + log_debug("[WS] Repling with Cache content for method \"%s\"", qPrintable(methodName)); + + if (pCacheItem->methodType == CacheMethodType::CACHE_METHOD) + { + writeToClient_(paramsHash, packetId, packetMsg.id, p.from, instanceId); + } + else if (pCacheItem->methodType == CacheMethodType::SUBSCRIBE_METHOD) + { + QByteArray subscriptionKey = paramsHash + "-sub"; + QByteArray updateCountKey = paramsHash + "-updateCount"; + QByteArray countBytes = redis_load_cache_response(updateCountKey); + int updateCount = countBytes.toInt(); + pCacheItem->subscriptionUpdateCount = updateCount; + writeToClient_(paramsHash, packetId, packetMsg.id, p.from, instanceId); + writeToClient_(subscriptionKey, packetId, packetMsg.id, p.from, instanceId); + // add client to list + pCacheItem->clientMap[packetId].msgId = msgIdStr; + pCacheItem->clientMap[packetId].from = p.from; + pCacheItem->clientMap[packetId].instanceId = instanceId; + log_debug("[WS] Adding new client id msgId=%s clientId=%s", qPrintable(msgIdStr), packetId.data()); + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); } } + else + { + log_debug("[WS] Already cache registered, but not added content \"%s\"", qPrintable(methodName)); + // add client to list + pCacheItem->clientMap[packetId].msgId = msgIdStr; + pCacheItem->clientMap[packetId].from = p.from; + pCacheItem->clientMap[packetId].instanceId = instanceId; + log_debug("[WS] Adding new client id msgId=%s clientId=%s", qPrintable(msgIdStr), packetId.data()); + pCacheItem->lastRefreshTime = QDateTime::currentMSecsSinceEpoch(); + } + return -1; } - + else { - QHashIterator > sit(serverSessionsBySender[n]); - while(sit.hasNext()) + // Register new cache item + int ccIndex = register_ws_cache_item(p, packetId, msgIdStr, methodName, msgParams, paramsHash); + if (ccIndex < 0) { - sit.next(); - const QByteArray &sender = sit.key(); - const QList &sessions = sit.value(); + log_warning("[WS] not initialized cache client, ignore"); + return 0; + } + log_debug("[WS] Registered New Cache Item for id=%s method=\"%s\"", qPrintable(msgIdStr), qPrintable(methodName)); + + pCacheItem = load_cache_item(paramsHash); + if (pCacheItem == NULL) + { + log_debug("[WS_REQ] not found cache item %s", paramsHash.toHex().data()); + return -1; + } + // Send new client cache request packet + pCacheItem->newMsgId = send_ws_request_over_cacheclient(p, msgIdStr, ccIndex); + pCacheItem->lastRequestTime = QDateTime::currentMSecsSinceEpoch(); - if(!sessions.isEmpty()) - { - QList ids; - foreach(KeepAliveRegistration *i, sessions) - { - assert(i->type == type); - if(type == HttpSession) - ids += ZhttpResponsePacket::Id(i->p.req->rid().second, i->p.req->outSeqInc()); - else // WebSocketSession - ids += ZhttpResponsePacket::Id(i->p.sock->rid().second, i->p.sock->outSeqInc()); - } + //store_cache_item_field(paramsHash, "lastRequestTime", pCacheItem->lastRequestTime); - writeKeepAlive(type, ids, sender); - } - } + // register cache refresh + register_cache_refresh(paramsHash, gWsCacheClientList[ccIndex].urlPath); } + + return -1; } - ++currentSessionRefreshBucket; - if(currentSessionRefreshBucket >= ZHTTP_REFRESH_BUCKETS) - currentSessionRefreshBucket = 0; + // log unhitted method + log_debug("[CACHE ITME] not hit method = %s", qPrintable(methodName)); + + return 0; } }; ZhttpManager::ZhttpManager(QObject *parent) : QObject(parent) { - d = new Private(this); + d = std::make_shared(this); } -ZhttpManager::~ZhttpManager() -{ - delete d; -} +ZhttpManager::~ZhttpManager() = default; int ZhttpManager::connectionCount() const { @@ -1270,4 +3517,329 @@ int ZhttpManager::estimateResponseHeaderBytes(int code, const QByteArray &reason return total; } +void initCacheClient(int workerNo) +{ + if (gWorkersCount == 1) + { + for (int i=0; i countMethodGroupMap + ) +{ + gWorkersCount++; + if (gCacheEnable == true) + { + log_debug("[CONFIG] already passed"); + return; + } + gCacheEnable = cacheEnable; + gHttpBackendUrlList = httpBackendUrlList; + gWsBackendUrlList = wsBackendUrlList; + + // method list + foreach (QString method, cacheMethodList) + { + gCacheMethodList.append(method.toLower()); + } + for (int i = 0; i < subscribeMethodList.count(); i++) + { + QStringList tmpList = subscribeMethodList[i].split(u'+'); + if (tmpList.count() == 2) + { + gSubscribeMethodMap[tmpList[0].toLower()] = tmpList[1]; + } + } + foreach (QString method, neverTimeoutMethodList) + { + gNeverTimeoutMethodList.append(method.toLower()); + } + foreach (QString method, refreshShorterMethodList) + { + gRefreshShorterMethodList.append(method.toLower()); + } + foreach (QString method, refreshLongerMethodList) + { + gRefreshLongerMethodList.append(method.toLower()); + } + foreach (QString method, refreshUneraseMethodList) + { + gRefreshUneraseMethodList.append(method.toLower()); + } + foreach (QString method, refreshExcludeMethodList) + { + gRefreshExcludeMethodList.append(method.toLower()); + } + foreach (QString method, refreshPassthroughMethodList) + { + gRefreshPassthroughMethodList.append(method.toLower()); + } + foreach (QString method, nullResponseMethodList) + { + gNullResponseMethodList.append(method.toLower()); + } + + // cache key item list + for (int i = 0; i < cacheKeyItemList.size(); ++i) + { + int lastDot = cacheKeyItemList[i].lastIndexOf('.'); + if (lastDot != -1) { + CacheKeyItem keyItem; + keyItem.keyName = cacheKeyItemList[i].left(lastDot); + QString flagVal = cacheKeyItemList[i].mid(lastDot + 1); + if (flagVal == "JSON_VALUE") + keyItem.flag = ItemFlag::JSON_VALUE; + else if (flagVal == "JSON_PAIR") + keyItem.flag = ItemFlag::JSON_PAIR; + else if (flagVal == "RAW_VALUE") + keyItem.flag = ItemFlag::RAW_VALUE; + else + continue; + + gCacheKeyItemList.append(keyItem); + } + else + { + continue; + } + } + + // attributes + gMsgIdAttrName = msgIdFieldName; + gMsgMethodAttrName = msgMethodFieldName; + gMsgParamsAttrName = msgParamsFieldName; + foreach (QString attr, msgErrorFieldList) + { + gErrorAttrList.append(attr.toLower()); + } + + log_debug("[CONFIG] cache %s", gCacheEnable ? "enabled" : "disabled"); + log_debug("[CONFIG] error attributes:"); + for (int i = 0; i < gErrorAttrList.size(); ++i) { + log_debug("%s", qPrintable(gErrorAttrList[i])); + } + + log_debug("[CONFIG] gHttpBackendUrlList"); + for (int i = 0; i < gHttpBackendUrlList.size(); ++i) { + QString connectPath = gHttpBackendUrlList[i]; + log_debug("%s", qPrintable(connectPath)); + httpCacheClientConnectFailedCountMap[connectPath] = 0; + httpCacheClientInvalidResponseCountMap[connectPath] = 0; + } + + log_debug("[CONFIG] gWsBackendUrlList"); + for (int i = 0; i < gWsBackendUrlList.size(); ++i) { + QString connectPath = gWsBackendUrlList[i]; + log_debug("%s", qPrintable(connectPath)); + wsCacheClientConnectFailedCountMap[connectPath] = 0; + wsCacheClientInvalidResponseCountMap[connectPath] = 0; + } + + log_debug("[CONFIG] gCacheMethodList"); + for (int i = 0; i < gCacheMethodList.size(); ++i) { + log_debug("%s", qPrintable(gCacheMethodList[i])); + } + + log_debug("[CONFIG] gSubscribeMethodMap"); + for (const auto &key : gSubscribeMethodMap.keys()) { + log_debug("%s:%s", qPrintable(key), qPrintable(gSubscribeMethodMap.value(key))); + } + + log_debug("[CONFIG] gNeverTimeoutMethodList"); + for (int i = 0; i < gNeverTimeoutMethodList.size(); ++i) { + log_debug("%s", qPrintable(gNeverTimeoutMethodList[i])); + } + + log_debug("[CONFIG] gRefreshShorterMethodList"); + for (int i = 0; i < gRefreshShorterMethodList.size(); ++i) { + log_debug("%s", qPrintable(gRefreshShorterMethodList[i])); + } + + log_debug("[CONFIG] gRefreshLongerMethodList"); + for (int i = 0; i < gRefreshLongerMethodList.size(); ++i) { + log_debug("%s", qPrintable(gRefreshLongerMethodList[i])); + } + + log_debug("[CONFIG] gRefreshUneraseMethodList"); + for (int i = 0; i < gRefreshUneraseMethodList.size(); ++i) { + log_debug("%s", qPrintable(gRefreshUneraseMethodList[i])); + } + + log_debug("[CONFIG] gRefreshExcludeMethodList"); + for (int i = 0; i < gRefreshExcludeMethodList.size(); ++i) { + log_debug("%s", qPrintable(gRefreshExcludeMethodList[i])); + } + + log_debug("[CONFIG] gRefreshPassthroughMethodList"); + for (int i = 0; i < gRefreshPassthroughMethodList.size(); ++i) { + log_debug("%s", qPrintable(gRefreshPassthroughMethodList[i])); + } + + log_debug("[CONFIG] gNullResponseMethodList"); + for (int i = 0; i < gNullResponseMethodList.size(); ++i) { + log_debug("%s", qPrintable(gNullResponseMethodList[i])); + } + + log_debug("[CONFIG] gCacheKeyItemList"); + for (int i = 0; i < gCacheKeyItemList.size(); ++i) { + log_debug("%s, %d", qPrintable(gCacheKeyItemList[i].keyName), gCacheKeyItemList[i].flag); + } + + log_debug("gMsgIdAttrName = %s", qPrintable(gMsgIdAttrName)); + log_debug("gMsgMethodAttrName = %s", qPrintable(gMsgMethodAttrName)); + log_debug("gMsgParamsAttrName = %s", qPrintable(gMsgParamsAttrName)); + + if (gCacheEnable == true) + { + if (gWsBackendUrlList.count() == 0) + { + log_debug("[WS] not defined ws backend url, exiting"); + exit(0); + } + QTimer::singleShot(2 * 1000, [=]() { + initCacheClient(0); + }); + + QTimer::singleShot(120 * 1000, [=]() { + check_cache_clients(); + }); + + gCacheThread = QtConcurrent::run(cache_thread); + } + + // timeout + gCacheTimeoutSeconds = cacheTimeoutSeconds; + gShorterTimeoutSeconds = shorterTimeoutSeconds; + gLongerTimeoutSeconds = longerTimeoutSeconds; + gAccessTimeoutSeconds = accessTimeoutSeconds; + + log_debug("gCacheTimeoutSeconds = %d", gCacheTimeoutSeconds); + log_debug("gShorterTimeoutSeconds = %d", gShorterTimeoutSeconds); + log_debug("gLongerTimeoutSeconds = %d", gLongerTimeoutSeconds); + log_debug("gAccessTimeoutSeconds = %d", gAccessTimeoutSeconds); + + // Maximum cache item countd + gCacheItemMaxCount = cacheItemMaxCount; + log_debug("gCacheItemMaxCount = %d", gCacheItemMaxCount); + + // time seconds to retry another backend for null response + gBackendSwitchIntervalSeconds = backendSwitchIntervalSeconds; + + // prometheus restore allow seconds (default 300) + gPrometheusRestoreAllowSeconds=prometheusRestoreAllowSeconds; + + // init redis + gRedisEnable = redisEnable; + gRedisHostAddr = redisHostAddr; + gRedisPort = redisPort; + gRedisPoolCount = redisPoolCount; + gRedisKeyHeader = redisKeyHeader; + gReplicaMasterAddr = replicaMasterAddr; + gReplicaMasterPort = replicaMasterPort; + log_debug("[CONFIG] redis %s, host=%s, port=%d, pool=%d, keyHeader=%s", gRedisEnable ? "enabled" : "disabled", + qPrintable(gRedisHostAddr), gRedisPort, gRedisPoolCount, qPrintable(gRedisKeyHeader)); + if (gRedisEnable == true) + { + if (gRedisHostAddr == "127.0.0.1") + redis_removeall_cache_item(); + + if (!gReplicaMasterAddr.isEmpty()) + { + gReplicaFlag = true; + redis_reset_replica(); + } + } + + // count method group + log_debug("[CONFIG] count method group"); + foreach(QString groupKey, countMethodGroupMap.keys()) + { + QString groupTotalStr = groupKey; + groupMethodCountMap[groupKey] = 0; + QStringList groupStrList = countMethodGroupMap[groupKey]; + groupTotalStr += ":"; + for (int i = 0; i < groupStrList.count(); i++) + groupTotalStr += groupStrList[i]+","; + log_debug("%s", qPrintable(groupTotalStr)); + gCountMethodGroupMap[groupKey] = groupStrList; + } + + //restore_prometheusStatFromFile(); +} + #include "zhttpmanager.moc" diff --git a/src/core/zhttpmanager.h b/src/core/zhttpmanager.h index 0994170c4..25140b470 100644 --- a/src/core/zhttpmanager.h +++ b/src/core/zhttpmanager.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2012-2013 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -61,6 +62,42 @@ class ZhttpManager : public QObject bool setServerInStreamSpecs(const QStringList &specs); bool setServerOutSpecs(const QStringList &specs); + void setCacheParameters( + bool cacheEnable, + const QStringList &httpBackendUrlList, + const QStringList &wsBackendUrlList, + const QStringList &cacheMethodList, + const QStringList &subscribeMethodList, + const QStringList &neverTimeoutMethodList, + const QStringList &refreshShorterMethodList, + const QStringList &refreshLongerMethodList, + const QStringList &refreshUneraseMethodList, + const QStringList &refreshExcludeMethodList, + const QStringList &refreshPassthroughMethodList, + const QStringList &nullResponseMethodList, + const QStringList &cacheKeyItemList, + const QString &msgIdFieldName, + const QString &msgMethodFieldName, + const QString &msgParamsFieldName, + const QStringList &msgErrorFieldList, + const int cacheTimeoutSeconds, + const int shorterTimeoutSeconds, + const int longerTimeoutSeconds, + const int accessTimeoutSeconds, + const int cacheItemMaxCount, + const int backendSwitchIntervalSeconds, + const int prometheusRestoreAllowSeconds, + bool redisEnable, + const QString &redisHostAddr, + const int redisPort, + const int redisPoolCount, + const QString &redisKeyHeader, + const QString &replicaMasterAddr, + const int replicaMasterPort, + QMap countMethodGroupMap); + + int create_wsCacheClientProcesses(); + ZhttpRequest *createRequest(); ZhttpRequest *takeNextRequest(); @@ -79,7 +116,7 @@ class ZhttpManager : public QObject private: class Private; friend class Private; - Private *d; + std::shared_ptr d; friend class ZhttpRequest; friend class ZWebSocket; diff --git a/src/core/zhttprequest.cpp b/src/core/zhttprequest.cpp index 6d5b4b593..6ec750abf 100644 --- a/src/core/zhttprequest.cpp +++ b/src/core/zhttprequest.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2012-2021 Fanout, Inc. - * Copyright (C) 2023 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -24,16 +24,16 @@ #include "zhttprequest.h" #include -#include #include "zhttprequestpacket.h" #include "zhttpresponsepacket.h" #include "bufferlist.h" #include "log.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "zhttpmanager.h" #include "uuidutil.h" -#define IDEAL_CREDITS 200000 +#define IDEAL_CREDITS 2000000 #define SESSION_EXPIRE 60000 #define KEEPALIVE_INTERVAL 45000 #define REQ_BUF_MAX 1000000 @@ -73,6 +73,7 @@ class ZhttpRequest::Private : public QObject bool ignorePolicies; bool trustConnectHost; bool ignoreTlsErrors; + int timeout; bool sendBodyAfterAck; QVariant passthrough; QString requestMethod; @@ -99,12 +100,15 @@ class ZhttpRequest::Private : public QObject bool writableChanged; bool errored; ErrorCondition errorCondition; - RTimer *expireTimer; - RTimer *keepAliveTimer; + Timer *expireTimer; + Timer *keepAliveTimer; + Timer *finishTimer; bool multi; bool quiet; Connection expTimerConnection; Connection keepAliveTimerConnection; + Connection finishTimerConnection; + DeferCall deferCall; Private(ZhttpRequest *_q) : QObject(_q), @@ -117,6 +121,7 @@ class ZhttpRequest::Private : public QObject ignorePolicies(false), trustConnectHost(false), ignoreTlsErrors(false), + timeout(0), sendBodyAfterAck(false), inSeq(0), outSeq(0), @@ -134,14 +139,15 @@ class ZhttpRequest::Private : public QObject errored(false), expireTimer(0), keepAliveTimer(0), + finishTimer(0), multi(false), quiet(false) { - expireTimer = new RTimer; + expireTimer = new Timer; expTimerConnection = expireTimer->timeout.connect(boost::bind(&Private::expire_timeout, this)); expireTimer->setSingleShot(true); - keepAliveTimer = new RTimer; + keepAliveTimer = new Timer; keepAliveTimerConnection = keepAliveTimer->timeout.connect(boost::bind(&Private::keepAlive_timeout, this)); } @@ -163,7 +169,7 @@ class ZhttpRequest::Private : public QObject { expTimerConnection.disconnect(); expireTimer->setParent(0); - expireTimer->deleteLater(); + DeferCall::deleteLater(expireTimer); expireTimer = 0; } @@ -171,10 +177,18 @@ class ZhttpRequest::Private : public QObject { keepAliveTimerConnection.disconnect(); keepAliveTimer->setParent(0); - keepAliveTimer->deleteLater(); + DeferCall::deleteLater(keepAliveTimer); keepAliveTimer = 0; } + if(finishTimer) + { + finishTimerConnection.disconnect(); + finishTimer->setParent(0); + DeferCall::deleteLater(finishTimer); + finishTimer = 0; + } + if(manager) { manager->unregisterKeepAlive(q); @@ -280,6 +294,14 @@ class ZhttpRequest::Private : public QObject { state = ClientStarting; + if(timeout > 0) + { + finishTimer = new Timer; + finishTimerConnection = finishTimer->timeout.connect(boost::bind(&Private::expire_timeout, this)); + finishTimer->setSingleShot(true); + finishTimer->start(timeout); + } + refreshTimeout(); update(); } @@ -367,7 +389,7 @@ class ZhttpRequest::Private : public QObject if(!pendingUpdate) { pendingUpdate = true; - QMetaObject::invokeMethod(this, "doUpdate", Qt::QueuedConnection); + deferCall.defer([&]() { doUpdate(); }); } } @@ -409,7 +431,7 @@ class ZhttpRequest::Private : public QObject void tryWrite() { - QPointer self = this; + std::weak_ptr self = q->d; if(state == ClientRequesting) { @@ -492,7 +514,7 @@ class ZhttpRequest::Private : public QObject } } - if(!self) + if(self.expired()) return; trySendPause(); @@ -935,7 +957,6 @@ class ZhttpRequest::Private : public QObject return ErrorGeneric; } -public slots: void doUpdate() { pendingUpdate = false; @@ -1044,9 +1065,9 @@ public slots: } else if(state == ClientRequesting) { - QPointer self = this; + std::weak_ptr self = q->d; tryWrite(); - if(!self) + if(self.expired()) return; if(writableChanged) @@ -1130,23 +1151,23 @@ public slots: cleanup(); } - QPointer self = this; + std::weak_ptr self = q->d; if(!packet.body.isEmpty()) q->bytesWritten(packet.body.size()); else if(!packet.more) q->bytesWritten(0); - if(!self) + if(self.expired()) return; trySendPause(); } else if(state == ServerResponding) { - QPointer self = this; + std::weak_ptr self = q->d; tryWrite(); - if(!self) + if(self.expired()) return; if(writableChanged) @@ -1157,7 +1178,6 @@ public slots: } } -public: void expire_timeout() { state = Stopped; @@ -1187,13 +1207,10 @@ public slots: ZhttpRequest::ZhttpRequest(QObject *parent) : HttpRequest(parent) { - d = new Private(this); + d = std::make_shared(this); } -ZhttpRequest::~ZhttpRequest() -{ - delete d; -} +ZhttpRequest::~ZhttpRequest() = default; ZhttpRequest::Rid ZhttpRequest::rid() const { @@ -1235,6 +1252,11 @@ void ZhttpRequest::setIgnoreTlsErrors(bool on) d->ignoreTlsErrors = on; } +void ZhttpRequest::setTimeout(int msecs) +{ + d->timeout = msecs; +} + void ZhttpRequest::setIsTls(bool on) { d->requestUri.setScheme(on ? "https" : "http"); diff --git a/src/core/zhttprequest.h b/src/core/zhttprequest.h index cefba914a..3f17edb28 100644 --- a/src/core/zhttprequest.h +++ b/src/core/zhttprequest.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2012-2016 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -27,6 +28,8 @@ #include "httprequest.h" #include +#define TIMERS_PER_ZHTTPREQUEST 3 + using Connection = boost::signals2::scoped_connection; class ZhttpRequestPacket; @@ -88,6 +91,7 @@ class ZhttpRequest : public HttpRequest virtual void setIgnorePolicies(bool on); virtual void setTrustConnectHost(bool on); virtual void setIgnoreTlsErrors(bool on); + virtual void setTimeout(int msecs); virtual void start(const QString &method, const QUrl &uri, const HttpHeaders &headers); virtual void beginResponse(int code, const QByteArray &reason, const HttpHeaders &headers); @@ -117,7 +121,7 @@ class ZhttpRequest : public HttpRequest private: class Private; friend class Private; - Private *d; + std::shared_ptr d; friend class ZhttpManager; ZhttpRequest(QObject *parent = 0); diff --git a/src/core/zrpcmanager.cpp b/src/core/zrpcmanager.cpp index 75fb12390..be0d56f7b 100644 --- a/src/core/zrpcmanager.cpp +++ b/src/core/zrpcmanager.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2016 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -63,10 +63,10 @@ class ZrpcManager::Private : public QObject int timeout; QStringList clientSpecs; QStringList serverSpecs; - QZmq::Socket *clientSock; - QZmq::Socket *serverSock; - QZmq::Valve *clientValve; - QZmq::Valve *serverValve; + std::unique_ptr clientSock; + std::unique_ptr serverSock; + std::unique_ptr clientValve; + std::unique_ptr serverValve; QHash clientReqsById; QList pending; Connection clientValveConnection; @@ -77,11 +77,7 @@ class ZrpcManager::Private : public QObject q(_q), ipcFileMode(-1), doBind(false), - timeout(-1), - clientSock(0), - serverSock(0), - clientValve(0), - serverValve(0) + timeout(-1) { } @@ -93,22 +89,22 @@ class ZrpcManager::Private : public QObject bool setupClient() { clientValveConnection.disconnect(); - delete clientValve; - delete clientSock; + clientValve.reset(); + clientSock.reset(); - clientSock = new QZmq::Socket(QZmq::Socket::Dealer, this); + clientSock = std::make_unique(QZmq::Socket::Dealer); clientSock->setSendHwm(OUT_HWM); clientSock->setShutdownWaitTime(REQ_WAIT_TIME); QString errorMessage; - if(!ZUtil::setupSocket(clientSock, clientSpecs, doBind, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(clientSock.get(), clientSpecs, doBind, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } - clientValve = new QZmq::Valve(clientSock, this); + clientValve = std::make_unique(clientSock.get()); clientValveConnection = clientValve->readyRead.connect(boost::bind(&Private::client_readyRead, this, boost::placeholders::_1)); clientValve->open(); @@ -119,22 +115,22 @@ class ZrpcManager::Private : public QObject bool setupServer() { serverValveConnection.disconnect(); - delete serverValve; - delete serverSock; + serverValve.reset(); + serverSock.reset(); - serverSock = new QZmq::Socket(QZmq::Socket::Router, this); + serverSock = std::make_unique(QZmq::Socket::Router); serverSock->setReceiveHwm(IN_HWM); serverSock->setShutdownWaitTime(REP_WAIT_TIME); QString errorMessage; - if(!ZUtil::setupSocket(serverSock, serverSpecs, doBind, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(serverSock.get(), serverSpecs, doBind, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } - serverValve = new QZmq::Valve(serverSock, this); + serverValve = std::make_unique(serverSock.get()); serverValveConnection = serverValve->readyRead.connect(boost::bind(&Private::server_readyRead, this, boost::placeholders::_1)); serverValve->open(); diff --git a/src/core/zrpcrequest.cpp b/src/core/zrpcrequest.cpp index 4c888c2eb..b792c8a25 100644 --- a/src/core/zrpcrequest.cpp +++ b/src/core/zrpcrequest.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2015 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -30,7 +30,8 @@ #include "zrpcmanager.h" #include "uuidutil.h" #include "log.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" using Connection = boost::signals2::scoped_connection; @@ -50,8 +51,9 @@ class ZrpcRequest::Private : public QObject QVariant result; ErrorCondition condition; QByteArray conditionString; - std::unique_ptr timer; + std::unique_ptr timer; Connection timerConnection; + DeferCall deferCall; Private(ZrpcRequest *_q) : QObject(_q), @@ -136,7 +138,6 @@ class ZrpcRequest::Private : public QObject q->finished(); } -private slots: void doStart() { if(!manager->canWriteImmediately()) @@ -156,7 +157,7 @@ private slots: if(manager->timeout() >= 0) { - timer = std::make_unique(); + timer = std::make_unique(); timerConnection = timer->timeout.connect(boost::bind(&Private::timer_timeout, this)); timer->setSingleShot(true); timer->start(manager->timeout()); @@ -238,7 +239,7 @@ void ZrpcRequest::start(const QString &method, const QVariantHash &args) { d->method = method; d->args = args; - QMetaObject::invokeMethod(d, "doStart", Qt::QueuedConnection); + d->deferCall.defer([=] { d->doStart(); }); } void ZrpcRequest::respond(const QVariant &result) diff --git a/src/core/zwebsocket.cpp b/src/core/zwebsocket.cpp index d6ee4e220..d590ff54c 100644 --- a/src/core/zwebsocket.cpp +++ b/src/core/zwebsocket.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2023 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -24,15 +24,15 @@ #include "zwebsocket.h" #include -#include #include "zhttprequestpacket.h" #include "zhttpresponsepacket.h" #include "log.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "zhttpmanager.h" #include "uuidutil.h" -#define IDEAL_CREDITS 200000 +#define IDEAL_CREDITS 2000000 #define SESSION_EXPIRE 60000 #define KEEPALIVE_INTERVAL 45000 @@ -84,8 +84,8 @@ class ZWebSocket::Private : public QObject bool readableChanged; bool writableChanged; ErrorCondition errorCondition; - RTimer *expireTimer; - RTimer *keepAliveTimer; + Timer *expireTimer; + Timer *keepAliveTimer; QList inFrames; QList outFrames; int inSize; @@ -95,6 +95,7 @@ class ZWebSocket::Private : public QObject bool multi; Connection expireTimerConnection; Connection keepAliveTimerConnection; + DeferCall deferCall; Private(ZWebSocket *_q) : QObject(_q), @@ -126,11 +127,11 @@ class ZWebSocket::Private : public QObject outContentType((int)Frame::Text), multi(false) { - expireTimer = new RTimer; + expireTimer = new Timer; expireTimerConnection = expireTimer->timeout.connect(boost::bind(&Private::expire_timeout, this)); expireTimer->setSingleShot(true); - keepAliveTimer = new RTimer; + keepAliveTimer = new Timer; keepAliveTimerConnection = keepAliveTimer->timeout.connect(boost::bind(&Private::keepAlive_timeout, this)); } @@ -151,7 +152,7 @@ class ZWebSocket::Private : public QObject { expireTimerConnection.disconnect(); expireTimer->setParent(0); - expireTimer->deleteLater(); + DeferCall::deleteLater(expireTimer); expireTimer = 0; } @@ -159,7 +160,7 @@ class ZWebSocket::Private : public QObject { keepAliveTimerConnection.disconnect(); keepAliveTimer->setParent(0); - keepAliveTimer->deleteLater(); + DeferCall::deleteLater(keepAliveTimer); keepAliveTimer = 0; } @@ -265,7 +266,7 @@ class ZWebSocket::Private : public QObject if(!pendingUpdate) { pendingUpdate = true; - QMetaObject::invokeMethod(this, "doUpdate", Qt::QueuedConnection); + deferCall.defer([=] { doUpdate(); }); } } @@ -297,7 +298,7 @@ class ZWebSocket::Private : public QObject state = Idle; cleanup(); - QMetaObject::invokeMethod(this, "doClosed", Qt::QueuedConnection); + deferCall.defer([=] { doClosed(); }); } Frame readFrame() @@ -337,7 +338,7 @@ class ZWebSocket::Private : public QObject // if peer was already closed, then we're done! state = Idle; cleanup(); - QMetaObject::invokeMethod(this, "doClosed", Qt::QueuedConnection); + deferCall.defer([=] { doClosed(); }); } else { @@ -349,7 +350,7 @@ class ZWebSocket::Private : public QObject void tryWrite() { - QPointer self = this; + std::weak_ptr self = q->d; if(state == Connected || state == ConnectedPeerClosed) { @@ -415,7 +416,7 @@ class ZWebSocket::Private : public QObject if(written > 0 || contentBytesWritten > 0) { q->framesWritten(written, contentBytesWritten); - if(!self) + if(self.expired()) return; } @@ -499,7 +500,7 @@ class ZWebSocket::Private : public QObject if(seq != inSeq) { - log_warning("zws server: error id=%s received message out of sequence, canceling", id.data()); + log_warning("zws server: error id=%s received message out of sequence (expected %d, got %d), canceling", id.data(), inSeq, seq); tryRespondCancel(packet); @@ -607,7 +608,7 @@ class ZWebSocket::Private : public QObject if(seq != inSeq) { - log_warning("zws client: error id=%s received message out of sequence, canceling", id.data()); + log_warning("zws client: error id=%s received message out of sequence (expected %d, got %d), canceling", id.data(), inSeq, seq); tryRespondCancel(packet); @@ -976,7 +977,6 @@ class ZWebSocket::Private : public QObject return ErrorGeneric; } -public slots: void doClosed() { q->closed(); @@ -999,10 +999,10 @@ public slots: } else { - QPointer self = this; + std::weak_ptr self = q->d; state = ConnectedPeerClosed; q->peerClosed(); - if(!self) + if(self.expired()) return; } } @@ -1012,9 +1012,9 @@ public slots: { readableChanged = false; - QPointer self = this; + std::weak_ptr self = q->d; q->readyRead(); - if(!self) + if(self.expired()) return; } } @@ -1052,9 +1052,9 @@ public slots: } else if(state == Connected || state == ConnectedPeerClosed) { - QPointer self = this; + std::weak_ptr self = q->d; tryWrite(); - if(!self) + if(self.expired()) return; if(writableChanged) @@ -1066,7 +1066,6 @@ public slots: } } -public: void expire_timeout() { state = Idle; @@ -1095,13 +1094,10 @@ public slots: ZWebSocket::ZWebSocket(QObject *parent) : WebSocket(parent) { - d = new Private(this); + d = std::make_shared(this); } -ZWebSocket::~ZWebSocket() -{ - delete d; -} +ZWebSocket::~ZWebSocket() = default; ZWebSocket::Rid ZWebSocket::rid() const { diff --git a/src/core/zwebsocket.h b/src/core/zwebsocket.h index c2114b1a3..a52c670de 100644 --- a/src/core/zwebsocket.h +++ b/src/core/zwebsocket.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2014-2016 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -80,7 +81,7 @@ class ZWebSocket : public WebSocket private: class Private; friend class Private; - Private *d; + std::shared_ptr d; friend class ZhttpManager; ZWebSocket(QObject *parent = 0); diff --git a/src/cpp.pro b/src/cpp.pro index e517395d8..b822871ef 100644 --- a/src/cpp.pro +++ b/src/cpp.pro @@ -1,8 +1,9 @@ TEMPLATE = lib CONFIG -= app_bundle -CONFIG += staticlib c++14 +CONFIG += staticlib c++17 QT -= gui QT += network +QT += concurrent TARGET = pushpin-cpp cpp_build_dir = $$OUT_PWD @@ -22,7 +23,9 @@ DEFINES += NO_IRISNET INCLUDEPATH += $$SRC_DIR/../target/include INCLUDEPATH += $$SRC_DIR/core -INCLUDEPATH += $$SRC_DIR/core/qzmq/src +INCLUDEPATH += /usr/include/hiredis + +LIBS += -lhiredis include(core/core.pri) include(m2adapter/m2adapter.pri) diff --git a/src/cpptests.pro b/src/cpptests.pro index 66a75d8b7..353e85ab5 100644 --- a/src/cpptests.pro +++ b/src/cpptests.pro @@ -1,6 +1,6 @@ TEMPLATE = lib CONFIG -= app_bundle -CONFIG += staticlib c++14 +CONFIG += staticlib c++17 QT -= gui QT *= network testlib TARGET = pushpin-cpptest @@ -18,7 +18,9 @@ DEFINES += NO_IRISNET INCLUDEPATH += $$SRC_DIR/../target/include INCLUDEPATH += $$SRC_DIR/core -INCLUDEPATH += $$SRC_DIR/core/qzmq/src +INCLUDEPATH += /usr/include/hiredis + +LIBS += -lhiredis include(core/tests.pri) include(proxy/tests.pri) diff --git a/src/handler/deferred.cpp b/src/handler/deferred.cpp index 60d309f85..deed5e1ee 100644 --- a/src/handler/deferred.cpp +++ b/src/handler/deferred.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2015 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -42,7 +43,7 @@ void Deferred::setFinished(bool ok, const QVariant &value) result_.success = ok; result_.value = value; - QMetaObject::invokeMethod(this, "doFinish", Qt::QueuedConnection); + deferCall_.defer([=] { doFinish(); }); } void Deferred::doFinish() diff --git a/src/handler/deferred.h b/src/handler/deferred.h index 127167685..2934d00ed 100644 --- a/src/handler/deferred.h +++ b/src/handler/deferred.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2015 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -26,6 +27,7 @@ #include #include #include +#include "defercall.h" class DeferredResult { @@ -63,11 +65,11 @@ class Deferred : public QObject void setFinished(bool ok, const QVariant &value = QVariant()); -private slots: - void doFinish(); - private: DeferredResult result_; + DeferCall deferCall_; + + void doFinish(); }; #endif diff --git a/src/handler/filter.cpp b/src/handler/filter.cpp index f2ea7614f..f200f4dd4 100644 --- a/src/handler/filter.cpp +++ b/src/handler/filter.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2016-2019 Fanout, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -22,13 +23,19 @@ #include "filter.h" +#include +#include #include "log.h" #include "format.h" #include "idformat.h" +#include "zhttpmanager.h" +#include "zhttprequest.h" + +#define REQUEST_TIMEOUT_SECS 10 namespace { -class SkipSelfFilter : public Filter +class SkipSelfFilter : public Filter, public Filter::MessageFilter { public: SkipSelfFilter() : @@ -36,6 +43,16 @@ class SkipSelfFilter : public Filter { } + virtual void start(const Filter::Context &context, const QByteArray &content) + { + setContext(context); + + Result r; + r.sendAction = sendAction(); + r.content = content; + finished(r); + } + virtual SendAction sendAction() const { QString user = context().subscriptionMeta.value("user"); @@ -47,7 +64,7 @@ class SkipSelfFilter : public Filter } }; -class SkipUsersFilter : public Filter +class SkipUsersFilter : public Filter, public Filter::MessageFilter { public: SkipUsersFilter() : @@ -55,6 +72,16 @@ class SkipUsersFilter : public Filter { } + virtual void start(const Filter::Context &context, const QByteArray &content) + { + setContext(context); + + Result r; + r.sendAction = sendAction(); + r.content = content; + finished(r); + } + virtual SendAction sendAction() const { QString user = context().subscriptionMeta.value("user"); @@ -74,7 +101,7 @@ class SkipUsersFilter : public Filter } }; -class RequireSubFilter : public Filter +class RequireSubFilter : public Filter, public Filter::MessageFilter { public: RequireSubFilter() : @@ -82,6 +109,16 @@ class RequireSubFilter : public Filter { } + virtual void start(const Filter::Context &context, const QByteArray &content) + { + setContext(context); + + Result r; + r.sendAction = sendAction(); + r.content = content; + finished(r); + } + virtual SendAction sendAction() const { QString require_sub = context().publishMeta.value("require_sub"); @@ -92,7 +129,7 @@ class RequireSubFilter : public Filter } }; -class BuildIdFilter : public Filter +class BuildIdFilter : public Filter, public Filter::MessageFilter { public: IdFormat::ContentRenderer *idContentRenderer; @@ -162,6 +199,16 @@ class BuildIdFilter : public Filter return true; } + virtual void start(const Filter::Context &context, const QByteArray &content) + { + setContext(context); + + Result r; + r.sendAction = sendAction(); + r.content = process(content); + finished(r); + } + virtual QByteArray update(const QByteArray &data) { if(!ensureInit()) @@ -223,7 +270,7 @@ class VarSubstFormatHandler : public Format::Handler } }; -class VarSubstFilter : public Filter +class VarSubstFilter : public Filter, public Filter::MessageFilter { public: VarSubstFilter() : @@ -231,6 +278,16 @@ class VarSubstFilter : public Filter { } + virtual void start(const Filter::Context &context, const QByteArray &content) + { + setContext(context); + + Result r; + r.sendAction = sendAction(); + r.content = process(content); + finished(r); + } + virtual QByteArray update(const QByteArray &data) { VarSubstFormatHandler handler; @@ -253,17 +310,339 @@ class VarSubstFilter : public Filter } }; +class HttpFilterInner; + +class HttpFilter : public Filter::MessageFilter +{ +public: + enum Mode + { + Check, + Modify + }; + + std::shared_ptr inner; + boost::signals2::scoped_connection finishedConnection; + + class RequestAction : public RateLimiter::Action + { + public: + std::weak_ptr inner; + + RequestAction(const std::shared_ptr &_inner) : + inner(_inner) + { + } + + virtual bool execute(); + }; + + HttpFilter(Mode mode); + + virtual void start(const Filter::Context &context, const QByteArray &content); + + void inner_finished(const Result &r); +}; + +class HttpFilterInner +{ +public: + HttpFilter::Mode mode; + std::unique_ptr req; + QUrl uri; + HttpHeaders headers; + QByteArray origContent; + bool haveResponseHeader; + QByteArray responseBody; + int responseSizeMax; + + boost::signals2::signal finished; + + HttpFilterInner(HttpFilter::Mode _mode) : + mode(_mode), + haveResponseHeader(false), + responseSizeMax(-1) + { + } + + void setup(ZhttpManager *zhttpOut, const QUrl &_uri, const HttpHeaders &_headers, const QVariant &passthroughData, const QByteArray &content, int _responseSizeMax) + { + uri = _uri; + headers = _headers; + origContent = content; + responseSizeMax = _responseSizeMax; + + req.reset(zhttpOut->createRequest()); + + // safe to not track, since req can't outlive this + req->readyRead.connect(boost::bind(&HttpFilterInner::req_readyRead, this)); + req->error.connect(boost::bind(&HttpFilterInner::req_error, this)); + + req->setPassthroughData(passthroughData); + } + + void startRequest() + { + // set timeout since filters should be fast + req->setTimeout(REQUEST_TIMEOUT_SECS * 1000); + + req->start("POST", uri, headers); + + if(mode == HttpFilter::Modify) + req->writeBody(origContent); + + req->endBody(); + } + + void req_readyRead() + { + if(!haveResponseHeader) + { + haveResponseHeader = true; + + int code = req->responseCode(); + switch(code) + { + case 200: + case 204: + break; + default: + Filter::MessageFilter::Result r; + r.errorMessage = QString("unexpected network request status: code=%1").arg(code); + doFinished(r); + return; + } + } + + QByteArray body = req->readBody(); + + if(mode == HttpFilter::Modify) + { + if(responseSizeMax >= 0 && responseBody.size() + body.size() > responseSizeMax) + { + Filter::MessageFilter::Result r; + r.errorMessage = QString("network response exceeded %1 bytes").arg(responseSizeMax); + doFinished(r); + return; + } + + responseBody += body; + } + + if(!req->isFinished()) + return; + + Filter::MessageFilter::Result r; + + if(req->responseHeaders().get("Action") == "drop") + { + // drop + r.sendAction = Filter::Drop; + } + else + { + // accept + r.sendAction = Filter::Send; + + switch(mode) + { + case HttpFilter::Check: + // as-is + r.content = origContent; + break; + case HttpFilter::Modify: + switch(req->responseCode()) + { + case 204: + // as-is + r.content = origContent; + break; + default: + // replace content + r.content = responseBody; + break; + } + break; + } + } + + doFinished(r); + } + + void req_error() + { + const char *s; + + switch(req->errorCondition()) + { + case HttpRequest::ErrorConnect: + s = "connection refused"; + break; + case HttpRequest::ErrorConnectTimeout: + s = "connection timed out"; + break; + case HttpRequest::ErrorTls: + s = "tls error"; + break; + case HttpRequest::ErrorDisconnected: + s = "disconnected"; + break; + case HttpRequest::ErrorTimeout: + s = "request timed out"; + break; + default: + s = "general error"; + break; + } + + Filter::MessageFilter::Result r; + r.errorMessage = QString("network request failed: %1").arg(s); + doFinished(r); + } + + void doFinished(const Filter::MessageFilter::Result &r) + { + req.reset(); + + finished(r); + } +}; + +bool HttpFilter::RequestAction::execute() +{ + auto target = inner.lock(); + if(!target) + return false; + + target->startRequest(); + return true; } -Filter::Filter(const QString &name) : - name_(name) +HttpFilter::HttpFilter(Mode mode) +{ + inner = std::make_shared(mode); + + finishedConnection = inner->finished.connect(boost::bind(&HttpFilter::inner_finished, this, boost::placeholders::_1)); +} + +void HttpFilter::start(const Filter::Context &context, const QByteArray &content) { + QUrl url = QUrl(context.subscriptionMeta.value("url"), QUrl::StrictMode); + if(!url.isValid()) + { + Result r; + r.errorMessage = "invalid or missing url value"; + finished(r); + return; + } + + QUrl currentUri = context.currentUri; + if(currentUri.scheme() == "wss") + currentUri.setScheme("https"); + else if(currentUri.scheme() == "ws") + currentUri.setScheme("http"); + + QUrl destUri = currentUri.resolved(url); + + int currentPort = currentUri.port(currentUri.scheme() == "https" ? 443 : 80); + int destPort = destUri.port(destUri.scheme() == "https" ? 443 : 80); + + QVariantHash passthroughData; + + passthroughData["route"] = context.route.toUtf8(); + + // if dest link points to the same service as the current request, + // then we can assume the network would send the request back to + // us, so we can handle it internally. if the link points to a + // different service, then we can't make this assumption and need + // to make the request over the network. note that such a request + // could still end up looping back to us + if(destUri.scheme() == currentUri.scheme() && destUri.host() == currentUri.host() && destPort == currentPort) + { + // tell the proxy that we prefer the request to be handled + // internally, using the same route + passthroughData["prefer-internal"] = true; + } + + // needed in case internal routing is not used + if(context.trusted) + passthroughData["trusted"] = true; + + HttpHeaders headers; + + { + QVariantMap vmap; + QHashIterator it(context.subscriptionMeta); + while(it.hasNext()) + { + it.next(); + vmap[it.key()] = it.value(); + } + + QJsonDocument doc = QJsonDocument(QJsonObject::fromVariantMap(vmap)); + headers += HttpHeader("Sub-Meta", doc.toJson(QJsonDocument::Compact)); + } + + { + QVariantMap vmap; + QHashIterator it(context.publishMeta); + while(it.hasNext()) + { + it.next(); + vmap[it.key()] = it.value(); + } + + QJsonDocument doc = QJsonDocument(QJsonObject::fromVariantMap(vmap)); + headers += HttpHeader("Pub-Meta", doc.toJson(QJsonDocument::Compact)); + } + + { + QHashIterator it(context.prevIds); + while(it.hasNext()) + { + it.next(); + const QString &name = it.key(); + const QString &prevId = it.value(); + + if(!prevId.isNull()) + headers += HttpHeader("Grip-Last", name.toUtf8() + "; last-id=" + prevId.toUtf8()); + } + } + + inner->setup(context.zhttpOut, destUri, headers, passthroughData, content, context.responseSizeMax); + + QString key = QString::fromUtf8(destUri.toEncoded()); + + assert(context.limiter); + + if(!context.limiter->addAction(key, new RequestAction(inner))) + { + // the limiter shouldn't have an hwm, but let's handle the error + // here in case one is ever added + + Result r; + r.errorMessage = "network request limit reached"; + finished(r); + return; + } } -Filter::~Filter() +void HttpFilter::inner_finished(const Result &r) { + finished(r); } +} + +Filter::MessageFilter::~MessageFilter() = default; + +Filter::Filter(const QString &name) : + name_(name) +{ +} + +Filter::~Filter() = default; + Filter::SendAction Filter::sendAction() const { return Send; @@ -308,6 +687,26 @@ Filter *Filter::create(const QString &name) return 0; } +Filter::MessageFilter *Filter::createMessageFilter(const QString &name) +{ + if(name == "skip-self") + return new SkipSelfFilter; + else if(name == "skip-users") + return new SkipUsersFilter; + else if(name == "require-sub") + return new RequireSubFilter; + else if(name == "build-id") + return new BuildIdFilter; + else if(name == "var-subst") + return new VarSubstFilter; + else if(name == "http-check") + return new HttpFilter(HttpFilter::Check); + else if(name == "http-modify") + return new HttpFilter(HttpFilter::Modify); + else + return 0; +} + QStringList Filter::names() { return (QStringList() @@ -315,7 +714,9 @@ QStringList Filter::names() << "skip-users" << "require-sub" << "build-id" - << "var-subst"); + << "var-subst" + << "http-check" + << "http-modify"); } Filter::Targets Filter::targets(const QString &name) @@ -327,9 +728,82 @@ Filter::Targets Filter::targets(const QString &name) else if(name == "require-sub") return Filter::MessageDelivery; else if(name == "build-id") - return Filter::Targets(Filter::MessageContent | Filter::ProxyContent); + return Filter::Targets(Filter::MessageContent | Filter::ResponseContent); else if(name == "var-subst") return Filter::MessageContent; + else if(name == "http-check") + return Filter::MessageDelivery; + else if(name == "http-modify") + return Filter::Targets(Filter::MessageDelivery | Filter::MessageContent); else return Filter::Targets(0); } + +Filter::MessageFilterStack::MessageFilterStack(const QStringList &filterNames) +{ + assert(filterNames.count() <= MESSAGEFILTERSTACK_SIZE_MAX); + + foreach(const QString &name, filterNames) + { + MessageFilter *f = createMessageFilter(name); + if(f) + filters_.emplace_back(std::unique_ptr(f)); + } +} + +void Filter::MessageFilterStack::start(const Filter::Context &context, const QByteArray &content) +{ + context_ = context; + content_ = content; + lastSendAction_ = Send; + + nextFilter(); +} + +void Filter::MessageFilterStack::nextFilter() +{ + if(filters_.empty()) + { + Result r; + r.sendAction = lastSendAction_; + r.content = content_; + finished(r); + return; + } + + finishedConnection_ = filters_.front()->finished.connect(boost::bind(&MessageFilterStack::filterFinished, this, boost::placeholders::_1)), + + // may call filterFinished immediately + filters_.front()->start(context_, content_); +} + +void Filter::MessageFilterStack::filterFinished(const Result &result) +{ + if(!result.errorMessage.isNull()) + { + filters_.clear(); + + Result r; + r.errorMessage = result.errorMessage; + finished(r); + return; + } + + lastSendAction_ = result.sendAction; + content_ = result.content; + + switch(lastSendAction_) + { + case Send: + // remove the finished filter + filters_.erase(filters_.begin()); + break; + case Drop: + // stop filtering. remove the finished filter and any remaining + filters_.clear(); + break; + } + + // will emit finished if there are no remaining filters + nextFilter(); +} diff --git a/src/handler/filter.h b/src/handler/filter.h index 48b872859..3f163f6ae 100644 --- a/src/handler/filter.h +++ b/src/handler/filter.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2016-2019 Fanout, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -26,6 +27,19 @@ #include #include #include +#include +#include +#include +#include "zhttprequest.h" +#include "ratelimiter.h" + +#define MESSAGEFILTERSTACK_SIZE_MAX 5 + +#define TIMERS_PER_MESSAGEFILTERSTACK (TIMERS_PER_ZHTTPREQUEST * MESSAGEFILTERSTACK_SIZE_MAX) + +#define DEFAULT_FILTER_RESPONSE_SIZE_MAX 100000 + +class ZhttpManager; class Filter { @@ -40,7 +54,7 @@ class Filter { MessageDelivery = 0x01, MessageContent = 0x02, - ProxyContent = 0x04, + ResponseContent = 0x04, }; class Context @@ -49,9 +63,61 @@ class Filter QHash prevIds; QHash subscriptionMeta; QHash publishMeta; + + // for network access + ZhttpManager *zhttpOut; + QUrl currentUri; + QString route; + bool trusted; + std::shared_ptr limiter; + int responseSizeMax; + + Context() : + zhttpOut(0), + trusted(false), + responseSizeMax(DEFAULT_FILTER_RESPONSE_SIZE_MAX) + { + } + }; + + class MessageFilter + { + public: + class Result + { + public: + SendAction sendAction; + QByteArray content; + QString errorMessage; // non-null on error + }; + + virtual ~MessageFilter(); + + // may emit finished immediately + virtual void start(const Filter::Context &context, const QByteArray &content = QByteArray()) = 0; + + boost::signals2::signal finished; + }; + + class MessageFilterStack : public MessageFilter + { + public: + MessageFilterStack(const QStringList &filterNames); + + // reimplemented + virtual void start(const Filter::Context &context, const QByteArray &content = QByteArray()); + + private: + std::vector> filters_; + Filter::Context context_; + QByteArray content_; + SendAction lastSendAction_; + boost::signals2::scoped_connection finishedConnection_; + + void nextFilter(); + void filterFinished(const Result &result); }; - Filter(const QString &name = QString()); virtual ~Filter(); const QString & name() const { return name_; } @@ -69,10 +135,13 @@ class Filter QByteArray process(const QByteArray &data); static Filter *create(const QString &name); + static MessageFilter *createMessageFilter(const QString &name); static QStringList names(); static Targets targets(const QString &name); protected: + Filter(const QString &name = QString()); + void setError(const QString &s) { errorMessage_ = s; } private: diff --git a/src/handler/filtertest.cpp b/src/handler/filtertest.cpp new file mode 100644 index 000000000..14a464b11 --- /dev/null +++ b/src/handler/filtertest.cpp @@ -0,0 +1,337 @@ +/* + * Copyright (C) 2025 Fastly, Inc. + * + * This file is part of Pushpin. + * + * $FANOUT_BEGIN_LICENSE:APACHE2$ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $FANOUT_END_LICENSE$ + */ + +#include +#include +#include +#include +#include +#include +#include "log.h" +#include "timer.h" +#include "defercall.h" +#include "zhttpmanager.h" +#include "ratelimiter.h" +#include "filter.h" + +class HttpFilterServer +{ +public: + std::unique_ptr zhttpIn; + std::unordered_map> reqs; + + HttpFilterServer(const QDir &workDir) + { + zhttpIn = std::make_unique(); + zhttpIn->setInstanceId("filter-test-server"); + zhttpIn->setBind(true); + zhttpIn->setServerInSpecs(QStringList() << QString("ipc://%1").arg(workDir.filePath("filter-test-in"))); + zhttpIn->setServerInStreamSpecs(QStringList() << QString("ipc://%1").arg(workDir.filePath("filter-test-in-stream"))); + zhttpIn->setServerOutSpecs(QStringList() << QString("ipc://%1").arg(workDir.filePath("filter-test-out"))); + zhttpIn->requestReady.connect(boost::bind(&HttpFilterServer::zhttpIn_requestReady, this)); + } + + void zhttpIn_requestReady() + { + ZhttpRequest *req = zhttpIn->takeNextRequest(); + if(!req) + return; + + req->readyRead.connect(boost::bind(&HttpFilterServer::req_readyRead, this, req)); + req->bytesWritten.connect(boost::bind(&HttpFilterServer::req_bytesWritten, this, req, boost::placeholders::_1)); + + reqs.emplace(std::make_pair(req, std::unique_ptr(req))); + + req_readyRead(req); + } + + void req_readyRead(ZhttpRequest *req) + { + if(!req->isInputFinished()) + return; + + handle(req); + } + + void req_bytesWritten(ZhttpRequest *req, int written) + { + Q_UNUSED(written); + + if(!req->isFinished()) + return; + + reqs.erase(req); + } + + void respond(ZhttpRequest *req, int code, const QByteArray &reason, const HttpHeaders &headers, const QByteArray &body) + { + req->beginResponse(code, reason, headers); + req->writeBody(body); + req->endBody(); + } + + void respondOk(ZhttpRequest *req, int code, bool accept, const QByteArray &body) + { + HttpHeaders headers; + if(!accept) + headers += HttpHeader("Action", "drop"); + + respond(req, code, "OK", headers, body); + } + + void respondError(ZhttpRequest *req, int code, const QByteArray &reason, const QByteArray &body) + { + respond(req, code, reason, HttpHeaders(), body); + } + + void handle(ZhttpRequest *req) + { + if(req->requestMethod() != "POST") + { + respondError(req, 400, "Bad Request", "Method must be POST\n"); + return; + } + + QUrl uri = req->requestUri(); + QByteArray body = req->readBody(); + + if(uri.path() == "/filter/accept") + { + respondOk(req, 200, true, ""); + } + else if(uri.path() == "/filter/drop") + { + respondOk(req, 200, false, ""); + } + else if(uri.path() == "/filter/modify") + { + if(req->requestHeaders().get("Grip-Last") != "test; last-id=a") + { + respondError(req, 400, "Bad Request", "Unexpected Grip-Last"); + return; + } + + QJsonDocument subMetaDoc = QJsonDocument::fromJson(req->requestHeaders().get("Sub-Meta")); + QJsonObject subMeta = subMetaDoc.object(); + QJsonDocument pubMetaDoc = QJsonDocument::fromJson(req->requestHeaders().get("Pub-Meta")); + QJsonObject pubMeta = pubMetaDoc.object(); + + QString prepend = subMeta["prepend"].toString(); + QString append = pubMeta["append"].toString(); + + if(!prepend.isEmpty() || !append.isEmpty()) + respondOk(req, 200, true, prepend.toUtf8() + body + append.toUtf8()); + else + respondOk(req, 204, true, ""); + } + else if(uri.path() == "/filter/large") + { + respondOk(req, 200, true, QByteArray(1001, 'a')); + } + else + { + respondError(req, 400, "Bad Request", "Bad Request\n"); + } + } +}; + +class FilterTest : public QObject +{ + Q_OBJECT + +private: + std::unique_ptr filterServer; + std::unique_ptr zhttpOut; + std::shared_ptr limiter; + + Filter::MessageFilter::Result runMessageFilters(const QStringList &filterNames, const Filter::Context &context, const QByteArray &content) + { + Filter::MessageFilterStack fs(filterNames); + + bool finished = false; + Filter::MessageFilter::Result r; + + fs.finished.connect([&](const Filter::MessageFilter::Result &_r) { + finished = true; + r = _r; + }); + + fs.start(context, content); + + while(!finished) + QTest::qWait(10); + + return r; + } + +private slots: + void initTestCase() + { + log_setOutputLevel(LOG_LEVEL_WARNING); + + QDir outDir(qgetenv("OUT_DIR")); + QDir workDir(QDir::current().relativeFilePath(outDir.filePath("test-work"))); + + Timer::init(100); + + filterServer = std::make_unique(workDir); + + zhttpOut = std::make_unique(); + zhttpOut->setInstanceId("filter-test-client"); + zhttpOut->setClientOutSpecs(QStringList() << QString("ipc://%1").arg(workDir.filePath("filter-test-in"))); + zhttpOut->setClientOutStreamSpecs(QStringList() << QString("ipc://%1").arg(workDir.filePath("filter-test-in-stream"))); + zhttpOut->setClientInSpecs(QStringList() << QString("ipc://%1").arg(workDir.filePath("filter-test-out"))); + + limiter = std::make_shared(); + + QTest::qWait(500); + } + + void cleanupTestCase() + { + zhttpOut.reset(); + filterServer.reset(); + + // ensure deferred deletes are processed + QCoreApplication::instance()->sendPostedEvents(); + + Timer::deinit(); + DeferCall::cleanup(); + } + + void messageFilters() + { + QStringList filterNames = QStringList() << "skip-self" << "var-subst"; + + Filter::Context context; + context.subscriptionMeta["user"] = "alice"; + + QByteArray content = "hello %(user)s"; + + { + auto r = runMessageFilters(filterNames, context, content); + QVERIFY(r.errorMessage.isNull()); + QCOMPARE(r.sendAction, Filter::Send); + QCOMPARE(r.content, "hello alice"); + } + + { + context.publishMeta["sender"] = "alice"; + auto r = runMessageFilters(filterNames, context, content); + QVERIFY(r.errorMessage.isNull()); + QCOMPARE(r.sendAction, Filter::Drop); + } + } + + void httpCheck() + { + QStringList filterNames = QStringList() << "http-check"; + + Filter::Context context; + context.subscriptionMeta["url"] = "/filter/accept"; + context.zhttpOut = zhttpOut.get(); + context.currentUri = "http://localhost/"; + context.limiter = limiter; + + QByteArray content = "hello world"; + + { + auto r = runMessageFilters(filterNames, context, content); + QVERIFY(r.errorMessage.isNull()); + QCOMPARE(r.sendAction, Filter::Send); + QCOMPARE(r.content, "hello world"); + } + + context.subscriptionMeta["url"] = "/filter/drop"; + + { + auto r = runMessageFilters(filterNames, context, content); + QVERIFY(r.errorMessage.isNull()); + QCOMPARE(r.sendAction, Filter::Drop); + } + + context.subscriptionMeta["url"] = "/filter/error"; + + { + auto r = runMessageFilters(filterNames, context, content); + QCOMPARE(r.errorMessage, "unexpected network request status: code=400"); + } + } + + void httpModify() + { + QStringList filterNames = QStringList() << "http-modify"; + + Filter::Context context; + context.prevIds["test"] = "a"; + context.subscriptionMeta["url"] = "/filter/modify"; + context.zhttpOut = zhttpOut.get(); + context.currentUri = "http://localhost/"; + context.limiter = limiter; + + QByteArray content = "hello world"; + + { + auto r = runMessageFilters(filterNames, context, content); + QVERIFY(r.errorMessage.isNull()); + QCOMPARE(r.sendAction, Filter::Send); + QCOMPARE(r.content, "hello world"); + } + + context.subscriptionMeta["prepend"] = "<<<"; + context.publishMeta["append"] = ">>>"; + + { + auto r = runMessageFilters(filterNames, context, content); + QVERIFY(r.errorMessage.isNull()); + QCOMPARE(r.sendAction, Filter::Send); + QCOMPARE(r.content, "<<>>"); + } + + context.subscriptionMeta.clear(); + context.publishMeta.clear(); + context.subscriptionMeta["url"] = "/filter/large"; + context.responseSizeMax = 1000; + + { + auto r = runMessageFilters(filterNames, context, content); + QCOMPARE(r.errorMessage, "network response exceeded 1000 bytes"); + } + } +}; + +namespace { +namespace Main { +QTEST_MAIN(FilterTest) +} +} + +extern "C" { + +int filter_test(int argc, char **argv) +{ + return Main::main(argc, argv); +} + +} + +#include "filtertest.moc" diff --git a/src/handler/handlerapp.cpp b/src/handler/handlerapp.cpp index e21522576..46d60f64d 100644 --- a/src/handler/handlerapp.cpp +++ b/src/handler/handlerapp.cpp @@ -268,7 +268,12 @@ class HandlerApp::Private : public QObject QStringList condure_out_specs = settings.value("proxy/condure_out_specs").toStringList(); trimlist(&condure_out_specs); connmgr_out_specs += condure_out_specs; + + bool cacheEnable = settings.value("cache/cache_enable").toBool(); + int proxyWorkerCount = settings.value("proxy/workers", 1).toInt(); + if (cacheEnable == true) + proxyWorkerCount = 1; QStringList m2a_in_stream_specs = settings.value("handler/m2a_in_stream_specs").toStringList(); trimlist(&m2a_in_stream_specs); QStringList m2a_out_specs = settings.value("handler/m2a_out_specs").toStringList(); diff --git a/src/handler/handlerengine.cpp b/src/handler/handlerengine.cpp index 650eb9ec2..b95ece6e3 100644 --- a/src/handler/handlerengine.cpp +++ b/src/handler/handlerengine.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2015-2023 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -25,8 +25,6 @@ #include #include -#include -#include #include #include #include @@ -36,7 +34,8 @@ #include "qzmqreqmessage.h" #include "qtcompat.h" #include "tnetstring.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "log.h" #include "logutil.h" #include "packet/httprequestdata.h" @@ -88,12 +87,6 @@ #define INSPECT_WORKERS_MAX 10 #define ACCEPT_WORKERS_MAX 10 -// each session can have a bunch of timers: -// 2 per incoming zhttprequest -// 2 per outgoing zhttprequest -// 2 per httpsession -#define TIMERS_PER_SESSION 10 - using namespace VariantUtil; static QList parseItems(const QVariantList &vitems, bool *ok = 0, QString *errorMessage = 0) @@ -399,8 +392,8 @@ class Subscription; class CommonState { public: - QHash httpSessions; - QHash wsSessions; + QHash> httpSessions; + QHash> wsSessions; QHash > responseSessionsByChannel; QHash > streamSessionsByChannel; QHash > wsSessionsByChannel; @@ -425,6 +418,7 @@ class AcceptWorker : public Deferred ZhttpManager *zhttpOut; StatsManager *stats; RateLimiter *updateLimiter; + std::shared_ptr filterLimiter; HttpSessionUpdateManager *httpSessionUpdateManager; QString route; QString statsRoute; @@ -441,12 +435,12 @@ class AcceptWorker : public Deferred bool responseSent; QString sid; LastIds lastIds; - QList sessions; + QList> sessions; int connectionSubscriptionMax; QSet needRemoveFromStats; map finishedConnection; - AcceptWorker(ZrpcRequest *_req, ZrpcManager *_stateClient, CommonState *_cs, ZhttpManager *_zhttpIn, ZhttpManager *_zhttpOut, StatsManager *_stats, RateLimiter *_updateLimiter, HttpSessionUpdateManager *_httpSessionUpdateManager, int _connectionSubscriptionMax, QObject *parent = 0) : + AcceptWorker(ZrpcRequest *_req, ZrpcManager *_stateClient, CommonState *_cs, ZhttpManager *_zhttpIn, ZhttpManager *_zhttpOut, StatsManager *_stats, RateLimiter *_updateLimiter, const std::shared_ptr &_filterLimiter, HttpSessionUpdateManager *_httpSessionUpdateManager, int _connectionSubscriptionMax, QObject *parent = 0) : Deferred(parent), req(_req), stateClient(_stateClient), @@ -455,6 +449,7 @@ class AcceptWorker : public Deferred zhttpOut(_zhttpOut), stats(_stats), updateLimiter(_updateLimiter), + filterLimiter(_filterLimiter), httpSessionUpdateManager(_httpSessionUpdateManager), logLevel(-1), trusted(false), @@ -829,12 +824,12 @@ class AcceptWorker : public Deferred afterSessionCalls(); } - QList takeSessions() + QList> takeSessions() { - QList out = sessions; + QList> out = sessions; sessions.clear(); - foreach(HttpSession *hs, out) + foreach(const std::shared_ptr &hs, out) hs->setParent(0); return out; @@ -930,13 +925,13 @@ class AcceptWorker : public Deferred if(!responseSent) { - // apply ProxyContent filters of all channels + // apply ResponseContent filters of all channels QStringList allFilters; foreach(const Instruct::Channel &c, instruct.channels) { foreach(const QString &filter, c.filters) { - if((Filter::targets(filter) & Filter::ProxyContent) && !allFilters.contains(filter)) + if((Filter::targets(filter) & Filter::ResponseContent) && !allFilters.contains(filter)) allFilters += filter; } } @@ -1132,7 +1127,7 @@ class AcceptWorker : public Deferred QByteArray cid = rid.first + ':' + rid.second; needRemoveFromStats.remove(cid); - sessions += new HttpSession(httpReq, adata, instruct, zhttpOut, stats, updateLimiter, &cs->publishLastIds, httpSessionUpdateManager, connectionSubscriptionMax, this); + sessions += std::make_shared(httpReq, adata, instruct, zhttpOut, stats, updateLimiter, filterLimiter, &cs->publishLastIds, httpSessionUpdateManager, connectionSubscriptionMax); } // engine should directly connect to this and register the holds @@ -1160,6 +1155,8 @@ class AcceptWorker : public Deferred } }; +#define TIMERS_PER_SUBSCRIPTION 1 + class Subscription : public QObject { Q_OBJECT @@ -1178,7 +1175,7 @@ class Subscription : public QObject timer_->stop(); timer_->disconnect(this); timer_->setParent(0); - timer_->deleteLater(); + DeferCall::deleteLater(timer_); } } @@ -1189,8 +1186,8 @@ class Subscription : public QObject void start() { - timer_ = new QTimer(this); - connect(timer_, &QTimer::timeout, this, &Subscription::timer_timeout); + timer_ = new Timer; + timer_->timeout.connect(boost::bind(&Subscription::timer_timeout, this)); timer_->setSingleShot(true); timer_->start(SUBSCRIBED_DELAY); } @@ -1199,9 +1196,8 @@ class Subscription : public QObject private: QString channel_; - QTimer *timer_; + Timer *timer_; -private slots: void timer_timeout() { subscribed(); @@ -1216,12 +1212,12 @@ class HandlerEngine::Private : public QObject class PublishAction : public RateLimiter::Action { public: - HandlerEngine::Private *ep; - QPointer target; + std::weak_ptr ep; + std::weak_ptr target; PublishItem item; QList exposeHeaders; - PublishAction(HandlerEngine::Private *_ep, QObject *_target, const PublishItem &_item, const QList &_exposeHeaders = QList()) : + PublishAction(const std::weak_ptr _ep, const std::weak_ptr _target, const PublishItem &_item, const QList &_exposeHeaders = QList()) : ep(_ep), target(_target), item(_item), @@ -1231,10 +1227,15 @@ class HandlerEngine::Private : public QObject virtual bool execute() { - if(!target) + auto epl = ep.lock(); + if(!epl) + return false; + + auto targetl = target.lock(); + if(!targetl) return false; - ep->publishSend(target, item, exposeHeaders); + epl->publishSend(targetl, item, exposeHeaders); return true; } }; @@ -1254,22 +1255,23 @@ class HandlerEngine::Private : public QObject ZrpcManager *stateClient; ZrpcManager *controlServer; ZrpcManager *proxyControlClient; - QZmq::Socket *inPullSock; - QZmq::Valve *inPullValve; - QZmq::Socket *inSubSock; - QZmq::Valve *inSubValve; - QZmq::Socket *retrySock; - QZmq::Socket *wsControlInitSock; - QZmq::Valve *wsControlInitValve; - QZmq::Socket *wsControlStreamSock; - QZmq::Valve *wsControlStreamValve; - QZmq::Socket *statsSock; - QZmq::Socket *proxyStatsSock; - QZmq::Valve *proxyStatsValve; + std::unique_ptr inPullSock; + std::unique_ptr inPullValve; + std::unique_ptr inSubSock; + std::unique_ptr inSubValve; + std::unique_ptr retrySock; + std::unique_ptr wsControlInitSock; + std::unique_ptr wsControlInitValve; + std::unique_ptr wsControlStreamSock; + std::unique_ptr wsControlStreamValve; + std::unique_ptr statsSock; + std::unique_ptr proxyStatsSock; + std::unique_ptr proxyStatsValve; SimpleHttpServer *controlHttpServer; StatsManager *stats; std::unique_ptr publishLimiter; std::unique_ptr updateLimiter; + std::shared_ptr filterLimiter; HttpSessionUpdateManager *httpSessionUpdateManager; Sequencer *sequencer; CommonState cs; @@ -1307,18 +1309,6 @@ class HandlerEngine::Private : public QObject stateClient(0), controlServer(0), proxyControlClient(0), - inPullSock(0), - inPullValve(0), - inSubSock(0), - inSubValve(0), - retrySock(0), - wsControlInitSock(0), - wsControlInitValve(0), - wsControlStreamSock(0), - wsControlStreamValve(0), - statsSock(0), - proxyStatsSock(0), - proxyStatsValve(0), controlHttpServer(0), stats(0), report(0) @@ -1327,6 +1317,7 @@ class HandlerEngine::Private : public QObject publishLimiter = std::make_unique(); updateLimiter = std::make_unique(); + filterLimiter = std::make_shared(); httpSessionUpdateManager = new HttpSessionUpdateManager(this); @@ -1339,8 +1330,8 @@ class HandlerEngine::Private : public QObject qDeleteAll(inspectWorkers); qDeleteAll(acceptWorkers); qDeleteAll(deferreds); - qDeleteAll(cs.wsSessions); - qDeleteAll(cs.httpSessions); + cs.wsSessions.clear(); + cs.httpSessions.clear(); qDeleteAll(cs.subs); } @@ -1348,8 +1339,13 @@ class HandlerEngine::Private : public QObject { config = _config; + // includes worst-case subscriptions and update registrations + int timersPerSession = qMax(TIMERS_PER_HTTPSESSION, TIMERS_PER_WSSESSION) + + (config.connectionSubscriptionMax * TIMERS_PER_SUBSCRIPTION) + + TIMERS_PER_UNIQUE_UPDATE_REGISTRATION; + // enough timers for sessions, plus an extra 100 for misc - RTimer::init((config.connectionsMax * TIMERS_PER_SESSION) + 100); + Timer::init((config.connectionsMax * timersPerSession) + 100); publishLimiter->setRate(config.messageRate); publishLimiter->setHwm(config.messageHwm); @@ -1357,6 +1353,8 @@ class HandlerEngine::Private : public QObject updateLimiter->setRate(10); updateLimiter->setBatchWaitEnabled(true); + filterLimiter->setRate(100); + sequencer->setWaitMax(config.messageWait); sequencer->setIdCacheTtl(config.idCacheTtl); @@ -1440,17 +1438,17 @@ class HandlerEngine::Private : public QObject if(!config.pushInSpec.isEmpty()) { - inPullSock = new QZmq::Socket(QZmq::Socket::Pull, this); + inPullSock = std::make_unique(QZmq::Socket::Pull); inPullSock->setHwm(DEFAULT_HWM); QString errorMessage; - if(!ZUtil::setupSocket(inPullSock, config.pushInSpec, true, config.ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(inPullSock.get(), config.pushInSpec, true, config.ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } - inPullValve = new QZmq::Valve(inPullSock, this); + inPullValve = std::make_unique(inPullSock.get()); pullConnection = inPullValve->readyRead.connect(boost::bind(&Private::inPull_readyRead, this, boost::placeholders::_1)); log_debug("in pull: %s", qPrintable(config.pushInSpec)); @@ -1458,12 +1456,12 @@ class HandlerEngine::Private : public QObject if(!config.pushInSubSpecs.isEmpty()) { - inSubSock = new QZmq::Socket(QZmq::Socket::Sub, this); + inSubSock = std::make_unique(QZmq::Socket::Sub); inSubSock->setSendHwm(SUB_SNDHWM); inSubSock->setShutdownWaitTime(0); QString errorMessage; - if(!ZUtil::setupSocket(inSubSock, config.pushInSubSpecs, !config.pushInSubConnect, config.ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(inSubSock.get(), config.pushInSubSpecs, !config.pushInSubConnect, config.ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; @@ -1477,7 +1475,7 @@ class HandlerEngine::Private : public QObject inSubSock->setTcpKeepAliveParameters(30, 6, 5); } - inSubValve = new QZmq::Valve(inSubSock, this); + inSubValve = std::make_unique(inSubSock.get()); inSubValveConnection = inSubValve->readyRead.connect(boost::bind(&Private::inSub_readyRead, this, boost::placeholders::_1)); log_debug("in sub: %s", qPrintable(config.pushInSubSpecs.join(", "))); @@ -1485,7 +1483,7 @@ class HandlerEngine::Private : public QObject if(!config.retryOutSpecs.isEmpty()) { - retrySock = new QZmq::Socket(QZmq::Socket::Router, this); + retrySock = std::make_unique(QZmq::Socket::Router); retrySock->setImmediateEnabled(true); retrySock->setHwm(DEFAULT_HWM); retrySock->setShutdownWaitTime(RETRY_WAIT_TIME); @@ -1494,7 +1492,7 @@ class HandlerEngine::Private : public QObject foreach(const QString &spec, config.retryOutSpecs) { QString errorMessage; - if(!ZUtil::setupSocket(retrySock, spec, false, config.ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(retrySock.get(), spec, false, config.ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; @@ -1506,25 +1504,25 @@ class HandlerEngine::Private : public QObject if(!config.wsControlInitSpecs.isEmpty() && !config.wsControlStreamSpecs.isEmpty()) { - wsControlInitSock = new QZmq::Socket(QZmq::Socket::Pull, this); + wsControlInitSock = std::make_unique(QZmq::Socket::Pull); wsControlInitSock->setHwm(DEFAULT_HWM); foreach(const QString &spec, config.wsControlInitSpecs) { QString errorMessage; - if(!ZUtil::setupSocket(wsControlInitSock, spec, false, config.ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(wsControlInitSock.get(), spec, false, config.ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } } - wsControlInitValve = new QZmq::Valve(wsControlInitSock, this); + wsControlInitValve = std::make_unique(wsControlInitSock.get()); controlInitValveConnection = wsControlInitValve->readyRead.connect(boost::bind(&Private::wsControlInit_readyRead, this, boost::placeholders::_1)); log_debug("ws control init: %s", qPrintable(config.wsControlInitSpecs.join(", "))); - wsControlStreamSock = new QZmq::Socket(QZmq::Socket::Router, this); + wsControlStreamSock = std::make_unique(QZmq::Socket::Router); wsControlStreamSock->setIdentity(config.instanceId); wsControlStreamSock->setImmediateEnabled(true); wsControlStreamSock->setHwm(DEFAULT_HWM); @@ -1533,14 +1531,14 @@ class HandlerEngine::Private : public QObject foreach(const QString &spec, config.wsControlStreamSpecs) { QString errorMessage; - if(!ZUtil::setupSocket(wsControlStreamSock, spec, false, config.ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(wsControlStreamSock.get(), spec, false, config.ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } } - wsControlStreamValve = new QZmq::Valve(wsControlStreamSock, this); + wsControlStreamValve = std::make_unique(wsControlStreamSock.get()); controlStreamValveConnection = wsControlStreamValve->readyRead.connect(boost::bind(&Private::wsControlStream_readyRead, this, boost::placeholders::_1)); log_debug("ws control stream: %s", qPrintable(config.wsControlStreamSpecs.join(", "))); @@ -1593,7 +1591,7 @@ class HandlerEngine::Private : public QObject if(!config.proxyStatsSpecs.isEmpty()) { - proxyStatsSock = new QZmq::Socket(QZmq::Socket::Sub, this); + proxyStatsSock = std::make_unique(QZmq::Socket::Sub); proxyStatsSock->setHwm(DEFAULT_HWM); proxyStatsSock->setShutdownWaitTime(0); proxyStatsSock->subscribe(""); @@ -1601,14 +1599,14 @@ class HandlerEngine::Private : public QObject foreach(const QString &spec, config.proxyStatsSpecs) { QString errorMessage; - if(!ZUtil::setupSocket(proxyStatsSock, spec, false, config.ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(proxyStatsSock.get(), spec, false, config.ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } } - proxyStatsValve = new QZmq::Valve(proxyStatsSock, this); + proxyStatsValve = std::make_unique(proxyStatsSock.get()); proxyStatConnection = proxyStatsValve->readyRead.connect(boost::bind(&Private::proxyStats_readyRead, this, boost::placeholders::_1)); log_debug("proxy stats: %s", qPrintable(config.proxyStatsSpecs.join(", "))); @@ -1756,9 +1754,8 @@ class HandlerEngine::Private : public QObject log_debug("removed ws session: %s", qPrintable(s->cid)); - cs.wsSessions.remove(s->cid); wsSessionConnectionMap.erase(s); - delete s; + cs.wsSessions.remove(s->cid); } void httpControlRespond(SimpleHttpRequest *req, int code, const QByteArray &reason, const QString &body, const QByteArray &contentType = QByteArray(), const HttpHeaders &headers = HttpHeaders(), int items = -1) @@ -1770,7 +1767,7 @@ class HandlerEngine::Private : public QObject outHeaders += HttpHeader("Content-Type", "text/plain"); req->respond(code, reason, outHeaders, body.toUtf8()); - req->finished.connect(boost::bind(&SimpleHttpRequest::deleteLater, req)); + req->finished.connect([=] { DeferCall::deleteLater(req); }); QString msg = QString("control: %1 %2 code=%3 %4").arg(req->requestMethod(), QString::fromUtf8(req->requestUri()), QString::number(code), QString::number(body.size())); if(items > -1) @@ -1779,88 +1776,12 @@ class HandlerEngine::Private : public QObject log_debug("%s", qPrintable(msg)); } - void publishSend(QObject *target, const PublishItem &item, const QList &exposeHeaders) + void publishSend(const std::shared_ptr &target, const PublishItem &item, const QList &exposeHeaders) { - const PublishFormat &f = item.format; - - if(f.type == PublishFormat::HttpResponse || f.type == PublishFormat::HttpStream) - { - HttpSession *hs = qobject_cast(target); - + if(auto hs = std::dynamic_pointer_cast(target)) hs->publish(item, exposeHeaders); - } - else if(f.type == PublishFormat::WebSocketMessage) - { - WsSession *s = qobject_cast(target); - - if(f.haveContentFilters) - { - // ensure content filters match - QStringList contentFilters; - foreach(const QString &f, s->channelFilters[item.channel]) - { - if(Filter::targets(f) & Filter::MessageContent) - contentFilters += f; - } - if(contentFilters != f.contentFilters) - { - QString errorMessage = QString("content filter mismatch: subscription=%1 message=%2").arg(contentFilters.join(","), f.contentFilters.join(",")); - log_debug("%s", qPrintable(errorMessage)); - return; - } - } - - Filter::Context fc; - fc.subscriptionMeta = s->meta; - fc.publishMeta = item.meta; - - FilterStack filters(fc, s->channelFilters[item.channel]); - - if(filters.sendAction() == Filter::Drop) - return; - - // TODO: hint support for websockets? - if(f.action != PublishFormat::Send && f.action != PublishFormat::Close && f.action != PublishFormat::Refresh) - return; - - WsControlPacket::Item i; - i.cid = s->cid.toUtf8(); - - if(f.action == PublishFormat::Send) - { - QByteArray body = filters.process(f.body); - if(body.isNull()) - { - log_debug("filter error: %s", qPrintable(filters.errorMessage())); - return; - } - - i.type = WsControlPacket::Item::Send; - - switch(f.messageType) - { - case PublishFormat::Text: i.contentType = "text"; break; - case PublishFormat::Binary: i.contentType = "binary"; break; - case PublishFormat::Ping: i.contentType = "ping"; break; - case PublishFormat::Pong: i.contentType = "pong"; break; - default: return; // unrecognized type, skip - } - - i.message = body; - } - else if(f.action == PublishFormat::Close) - { - i.type = WsControlPacket::Item::Close; - i.code = f.code; - i.reason = f.reason; - } - else if(f.action == PublishFormat::Refresh) - { - i.type = WsControlPacket::Item::Refresh; - } - - writeWsControlItems(s->peer, QList() << i); - } + else if(auto s = std::dynamic_pointer_cast(target)) + s->publish(item); } int blocksForData(int size) const @@ -1885,7 +1806,7 @@ class HandlerEngine::Private : public QObject } else { - foreach(HttpSession *hs, cs.httpSessions) + foreach(const std::shared_ptr &hs, cs.httpSessions) hs->update(); } } @@ -2019,7 +1940,7 @@ class HandlerEngine::Private : public QObject // accept request immediately before returning to the event loop. // the start() call will do this - AcceptWorker *w = new AcceptWorker(req, stateClient, &cs, zhttpIn, zhttpOut, stats, updateLimiter.get(), httpSessionUpdateManager, config.connectionSubscriptionMax, this); + AcceptWorker *w = new AcceptWorker(req, stateClient, &cs, zhttpIn, zhttpOut, stats, updateLimiter.get(), filterLimiter, httpSessionUpdateManager, config.connectionSubscriptionMax, this); finishedConnection[w] = w->finished.connect(boost::bind(&Private::acceptWorker_finished, this, boost::placeholders::_1, w)); sessionsReadyConnection[w] = w->sessionsReady.connect(boost::bind(&Private::acceptWorker_sessionsReady, this, w)); retryPacketReadyConnection[w] = w->retryPacketReady.connect(boost::bind(&Private::acceptWorker_retryPacketReady, this, boost::placeholders::_1, boost::placeholders::_2)); @@ -2244,11 +2165,13 @@ class HandlerEngine::Private : public QObject else blocks = blocksForData(f.body.size()); - foreach(HttpSession *hs, responseSessions) + foreach(HttpSession *hsp, responseSessions) { + std::shared_ptr &hs = cs.httpSessions[hsp->rid()]; + QString statsRoute = hs->statsRoute(); - if(!publishLimiter->addAction(statsRoute, new PublishAction(this, hs, i, exposeHeaders), blocks != -1 ? blocks : 1)) + if(!publishLimiter->addAction(statsRoute, new PublishAction(q->d, hs, i, exposeHeaders), blocks != -1 ? blocks : 1)) { if(!statsRoute.isEmpty()) log_warning("exceeded publish hwm (%d) for route %s, dropping message", config.messageHwm, qPrintable(statsRoute)); @@ -2278,11 +2201,13 @@ class HandlerEngine::Private : public QObject else blocks = blocksForData(f.body.size()); - foreach(HttpSession *hs, streamSessions) + foreach(HttpSession *hsp, streamSessions) { + std::shared_ptr &hs = cs.httpSessions[hsp->rid()]; + QString statsRoute = hs->statsRoute(); - if(!publishLimiter->addAction(statsRoute, new PublishAction(this, hs, i), blocks != -1 ? blocks : 1)) + if(!publishLimiter->addAction(statsRoute, new PublishAction(q->d, hs, i), blocks != -1 ? blocks : 1)) { if(!statsRoute.isEmpty()) log_warning("exceeded publish hwm (%d) for route %s, dropping message", config.messageHwm, qPrintable(statsRoute)); @@ -2312,11 +2237,13 @@ class HandlerEngine::Private : public QObject else blocks = blocksForData(f.body.size()); - foreach(WsSession *s, wsSessions) + foreach(WsSession *sp, wsSessions) { + std::shared_ptr &s = cs.wsSessions[sp->cid]; + QString statsRoute = s->statsRoute; - if(!publishLimiter->addAction(statsRoute, new PublishAction(this, s, i), blocks != -1 ? blocks : 1)) + if(!publishLimiter->addAction(statsRoute, new PublishAction(q->d, s, i), blocks != -1 ? blocks : 1)) { if(!statsRoute.isEmpty()) log_warning("exceeded publish hwm (%d) for route %s, dropping message", config.messageHwm, qPrintable(statsRoute)); @@ -2421,8 +2348,8 @@ class HandlerEngine::Private : public QObject void acceptWorker_sessionsReady(AcceptWorker *w) { - QList sessions = w->takeSessions(); - foreach(HttpSession *hs, sessions) + QList> sessions = w->takeSessions(); + foreach(const std::shared_ptr &hs, sessions) { // NOTE: for performance reasons we do not call hs->setParent and // instead leave the object unparented @@ -2454,7 +2381,7 @@ class HandlerEngine::Private : public QObject assert(at != -1); ZhttpRequest::Rid rid(id.mid(0, at), id.mid(at + 1)); - HttpSession *hs = cs.httpSessions.value(rid); + HttpSession *hs = cs.httpSessions.value(rid).get(); if(hs && !hs->sid().isEmpty()) sidLastIds[hs->sid()] = LastIds(); } @@ -2681,26 +2608,30 @@ class HandlerEngine::Private : public QObject if(item.type == WsControlPacket::Item::Here) { - WsSession *s = cs.wsSessions.value(item.cid); + std::shared_ptr s = cs.wsSessions.value(item.cid); if(!s) { - s = new WsSession(this); - wsSessionConnectionMap[s] = { - s->send.connect(boost::bind(&Private::wssession_send, this, boost::placeholders::_1, boost::placeholders::_2, boost::placeholders::_3, s)), - s->expired.connect(boost::bind(&Private::wssession_expired, this, s)), - s->error.connect(boost::bind(&Private::wssession_error, this, s)) + s = std::make_shared(); + wsSessionConnectionMap[s.get()] = { + s->send.connect(boost::bind(&Private::wssession_send, this, boost::placeholders::_1, s.get())), + s->expired.connect(boost::bind(&Private::wssession_expired, this, s.get())), + s->error.connect(boost::bind(&Private::wssession_error, this, s.get())) }; s->peer = packet.from; s->cid = QString::fromUtf8(item.cid); s->ttl = item.ttl; s->requestData.uri = item.uri; + s->zhttpOut = zhttpOut; + s->filterLimiter = filterLimiter; s->refreshExpiration(); cs.wsSessions.insert(s->cid, s); log_debug("added ws session: %s", qPrintable(s->cid)); } + s->debug = item.debug; s->route = item.route; s->statsRoute = item.separateStats ? item.route : QString(); + s->targetTrusted = item.trusted; s->channelPrefix = QString::fromUtf8(item.channelPrefix); if(item.logLevel >= 0) s->logLevel = item.logLevel; @@ -2712,7 +2643,7 @@ class HandlerEngine::Private : public QObject } // any other type must be for a known cid - WsSession *s = cs.wsSessions.value(QString::fromUtf8(item.cid)); + WsSession *s = cs.wsSessions.value(QString::fromUtf8(item.cid)).get(); if(!s) { // send cancel, causing the proxy to close the connection. client @@ -2740,7 +2671,7 @@ class HandlerEngine::Private : public QObject if(e.error != QJsonParseError::NoError || (!doc.isObject() && !doc.isArray())) { log_debug("grip control message is not valid json"); - return; + continue; } if(doc.isObject()) @@ -2753,13 +2684,19 @@ class HandlerEngine::Private : public QObject if(!ok) { log_debug("failed to parse grip control message: %s", qPrintable(errorMessage)); - return; + continue; } if(cm.type == WsControlMessage::Subscribe) { if(s->channels.count() < config.connectionSubscriptionMax) { + if(cm.filters.count() > MESSAGEFILTERSTACK_SIZE_MAX) + { + s->sendCloseError(QString("too many filters for channel '%1'").arg(cm.channel)); + continue; + } + QString channel = s->channelPrefix + cm.channel; s->channels += channel; s->channelFilters[channel] = cm.filters; @@ -3164,7 +3101,6 @@ class HandlerEngine::Private : public QObject } } -private slots: void hs_subscribe(HttpSession *hs, const QString &channel) { Instruct::HoldMode mode = hs->holdMode(); @@ -3208,32 +3144,24 @@ private slots: removeSessionChannel(hs, channel); } - void hs_finished(HttpSession *hs) + void hs_finished(HttpSession *hsp) { - QByteArray addr = hs->retryToAddress(); - RetryRequestPacket rp = hs->retryPacket(); + QByteArray addr = hsp->retryToAddress(); + RetryRequestPacket rp = hsp->retryPacket(); - cs.httpSessions.remove(hs->rid()); + std::shared_ptr hs = cs.httpSessions.take(hsp->rid()); hs->subscribeCallback().remove(this); hs->unsubscribeCallback().remove(this); hs->finishedCallback().remove(this); - hs->deleteLater(); + DeferCall::deleteLater(new std::shared_ptr(hs)); if(!rp.requests.isEmpty()) writeRetryPacket(addr, rp); } - void wssession_send(int reqId, const QByteArray &type, const QByteArray &message, WsSession *s) + void wssession_send(const WsControlPacket::Item &i, WsSession *s) { - WsControlPacket::Item i; - i.cid = s->cid.toUtf8(); - i.requestId = QByteArray::number(reqId); - i.type = WsControlPacket::Item::Send; - i.contentType = type; - i.message = message; - i.queue = true; - writeWsControlItems(s->peer, QList() << i); } @@ -3259,13 +3187,10 @@ private slots: HandlerEngine::HandlerEngine(QObject *parent) : QObject(parent) { - d = new Private(this); + d = std::make_shared(this); } -HandlerEngine::~HandlerEngine() -{ - delete d; -} +HandlerEngine::~HandlerEngine() = default; bool HandlerEngine::start(const Configuration &config) { diff --git a/src/handler/handlerengine.h b/src/handler/handlerengine.h index d6a1cf4e2..3743d9541 100644 --- a/src/handler/handlerengine.h +++ b/src/handler/handlerengine.h @@ -115,7 +115,7 @@ class HandlerEngine : public QObject private: class Private; - Private *d; + std::shared_ptr d; }; #endif diff --git a/src/handler/handlerenginetest.cpp b/src/handler/handlerenginetest.cpp index 27968ab9b..0db992acb 100644 --- a/src/handler/handlerenginetest.cpp +++ b/src/handler/handlerenginetest.cpp @@ -32,7 +32,8 @@ #include "zhttprequestpacket.h" #include "zhttpresponsepacket.h" #include "packet/httpresponsedata.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "handlerengine.h" namespace { @@ -42,17 +43,17 @@ class Wrapper : public QObject Q_OBJECT public: - QZmq::Socket *zhttpClientOutStreamSock; - QZmq::Socket *zhttpClientInSock; - QZmq::Valve *zhttpClientInValve; - QZmq::Socket *zhttpServerInSock; - QZmq::Valve *zhttpServerInValve; - QZmq::Socket *zhttpServerInStreamSock; - QZmq::Valve *zhttpServerInStreamValve; - QZmq::Socket *zhttpServerOutSock; - QZmq::Socket *proxyAcceptSock; - QZmq::Valve *proxyAcceptValve; - QZmq::Socket *publishPushSock; + std::unique_ptr zhttpClientOutStreamSock; + std::unique_ptr zhttpClientInSock; + std::unique_ptr zhttpClientInValve; + std::unique_ptr zhttpServerInSock; + std::unique_ptr zhttpServerInValve; + std::unique_ptr zhttpServerInStreamSock; + std::unique_ptr zhttpServerInStreamValve; + std::unique_ptr zhttpServerOutSock; + std::unique_ptr proxyAcceptSock; + std::unique_ptr proxyAcceptValve; + std::unique_ptr publishPushSock; QDir workDir; bool acceptSuccess; @@ -79,32 +80,32 @@ class Wrapper : public QObject { // http sockets - zhttpClientOutStreamSock = new QZmq::Socket(QZmq::Socket::Router, this); + zhttpClientOutStreamSock = std::make_unique(QZmq::Socket::Router); - zhttpClientInSock = new QZmq::Socket(QZmq::Socket::Sub, this); - zhttpClientInValve = new QZmq::Valve(zhttpClientInSock, this); + zhttpClientInSock = std::make_unique(QZmq::Socket::Sub); + zhttpClientInValve = std::make_unique(zhttpClientInSock.get()); zhttpClientInValveConnection = zhttpClientInValve->readyRead.connect(boost::bind(&Wrapper::zhttpClientIn_readyRead, this, boost::placeholders::_1)); - zhttpServerInSock = new QZmq::Socket(QZmq::Socket::Pull, this); - zhttpServerInValve = new QZmq::Valve(zhttpServerInSock, this); + zhttpServerInSock = std::make_unique(QZmq::Socket::Pull); + zhttpServerInValve = std::make_unique(zhttpServerInSock.get()); zhttpServerInValveConnection = zhttpServerInValve->readyRead.connect(boost::bind(&Wrapper::zhttpServerIn_readyRead, this, boost::placeholders::_1)); - zhttpServerInStreamSock = new QZmq::Socket(QZmq::Socket::Router, this); + zhttpServerInStreamSock = std::make_unique(QZmq::Socket::Router); zhttpServerInStreamSock->setIdentity("test-server"); - zhttpServerInStreamValve = new QZmq::Valve(zhttpServerInStreamSock, this); + zhttpServerInStreamValve = std::make_unique(zhttpServerInStreamSock.get()); zhttpServerInStreamValveConnection = zhttpServerInStreamValve->readyRead.connect(boost::bind(&Wrapper::zhttpServerInStream_readyRead, this, boost::placeholders::_1)); - zhttpServerOutSock = new QZmq::Socket(QZmq::Socket::Pub, this); + zhttpServerOutSock = std::make_unique(QZmq::Socket::Pub); // proxy sockets - proxyAcceptSock = new QZmq::Socket(QZmq::Socket::Dealer, this); - proxyAcceptValve = new QZmq::Valve(proxyAcceptSock, this); + proxyAcceptSock = std::make_unique(QZmq::Socket::Dealer); + proxyAcceptValve = std::make_unique(proxyAcceptSock.get()); proxyAcceptValveConnection = proxyAcceptValve->readyRead.connect(boost::bind(&Wrapper::proxyAccept_readyRead, this, boost::placeholders::_1)); // publish sockets - publishPushSock = new QZmq::Socket(QZmq::Socket::Push, this); + publishPushSock = std::make_unique(QZmq::Socket::Push); } void startHttp() @@ -308,7 +309,11 @@ private slots: delete engine; delete wrapper; - RTimer::deinit(); + // ensure deferred deletes are processed + QCoreApplication::instance()->sendPostedEvents(); + + Timer::deinit(); + DeferCall::cleanup(); } void acceptNoHold() diff --git a/src/handler/handlermain.cpp b/src/handler/handlermain.cpp index 3278619af..f3c11d43d 100644 --- a/src/handler/handlermain.cpp +++ b/src/handler/handlermain.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2016 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -22,8 +22,8 @@ */ #include -#include -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "handlerapp.h" class HandlerAppMain @@ -52,14 +52,16 @@ int handler_main(int argc, char **argv) QCoreApplication qapp(argc, argv); HandlerAppMain appMain; - QTimer::singleShot(0, [&appMain]() {appMain.start();}); + DeferCall deferCall; + deferCall.defer([&] { appMain.start(); }); int ret = qapp.exec(); // ensure deferred deletes are processed QCoreApplication::instance()->sendPostedEvents(); // deinit here, after all event loop activity has completed - RTimer::deinit(); + Timer::deinit(); + DeferCall::cleanup(); return ret; } diff --git a/src/handler/handlertests.h b/src/handler/handlertests.h index 79dbcdeb9..e891e9451 100644 --- a/src/handler/handlertests.h +++ b/src/handler/handlertests.h @@ -1,6 +1,7 @@ #ifndef HANDLER_TEST_H #define HANDLER_TEST_H +int filter_test(int argc, char **argv); int jsonpatch_test(int argc, char **argv); int instruct_test(int argc, char **argv); int idformat_test(int argc, char **argv); diff --git a/src/handler/httpsession.cpp b/src/handler/httpsession.cpp index a1e12cd30..61dff8b11 100644 --- a/src/handler/httpsession.cpp +++ b/src/handler/httpsession.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2016-2023 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -24,13 +24,13 @@ #include "httpsession.h" #include -#include #include #include #include #include #include "qtcompat.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "log.h" #include "bufferlist.h" #include "packet/retryrequestpacket.h" @@ -142,6 +142,19 @@ class HttpSession::Private : public QObject } }; + class QueuedItem + { + public: + PublishItem item; + QList exposeHeaders; + + QueuedItem(const PublishItem &_item, const QList &_exposeHeaders = QList()) : + item(_item), + exposeHeaders(_exposeHeaders) + { + } + }; + friend class UpdateAction; HttpSession *q; @@ -151,12 +164,13 @@ class HttpSession::Private : public QObject Instruct instruct; int logLevel; QHash channels; - RTimer *timer; - RTimer *retryTimer; + Timer *timer; + Timer *retryTimer; StatsManager *stats; ZhttpManager *outZhttp; std::unique_ptr outReq; // for fetching links RateLimiter *updateLimiter; + std::shared_ptr filterLimiter; PublishLastIds *publishLastIds; HttpSessionUpdateManager *updateManager; BufferList firstInstructResponse; @@ -170,10 +184,12 @@ class HttpSession::Private : public QObject bool needUpdate; Priority needUpdatePriority; UpdateAction *pendingAction; - QList publishQueue; + QList publishQueue; + bool inProcessPublishQueue; QByteArray retryToAddress; RetryRequestPacket retryPacket; LogUtil::Config logConfig; + std::unique_ptr messageFilters; FilterStack *responseFilters; QSet activeChannels; int connectionSubscriptionMax; @@ -189,8 +205,10 @@ class HttpSession::Private : public QObject Connection errorOutConnection; Connection timerConnection; Connection retryTimerConnection; + Connection messageFiltersFinishedConnection; + DeferCall deferCall; - Private(HttpSession *_q, ZhttpRequest *_req, const HttpSession::AcceptData &_adata, const Instruct &_instruct, ZhttpManager *_outZhttp, StatsManager *_stats, RateLimiter *_updateLimiter, PublishLastIds *_publishLastIds, HttpSessionUpdateManager *_updateManager, int _connectionSubscriptionMax) : + Private(HttpSession *_q, ZhttpRequest *_req, const HttpSession::AcceptData &_adata, const Instruct &_instruct, ZhttpManager *_outZhttp, StatsManager *_stats, RateLimiter *_updateLimiter, const std::shared_ptr _filterLimiter, PublishLastIds *_publishLastIds, HttpSessionUpdateManager *_updateManager, int _connectionSubscriptionMax) : QObject(_q), q(_q), req(_req), @@ -198,6 +216,7 @@ class HttpSession::Private : public QObject stats(_stats), outZhttp(_outZhttp), updateLimiter(_updateLimiter), + filterLimiter(_filterLimiter), publishLastIds(_publishLastIds), updateManager(_updateManager), haveOutReqHeaders(false), @@ -205,6 +224,7 @@ class HttpSession::Private : public QObject retries(0), needUpdate(false), pendingAction(0), + inProcessPublishQueue(false), responseFilters(0), connectionSubscriptionMax(_connectionSubscriptionMax), connectionStatsActive(true) @@ -216,10 +236,10 @@ class HttpSession::Private : public QObject writeBytesChangedConnection = req->writeBytesChanged.connect(boost::bind(&Private::req_writeBytesChanged, this)); errorConnection = req->error.connect(boost::bind(&Private::req_error, this)); - timer = new RTimer; + timer = new Timer; timerConnection = timer->timeout.connect(boost::bind(&Private::timer_timeout, this)); - retryTimer = new RTimer; + retryTimer = new Timer; retryTimerConnection = retryTimer->timeout.connect(boost::bind(&Private::retryTimer_timeout, this)); retryTimer->setSingleShot(true); @@ -247,11 +267,11 @@ class HttpSession::Private : public QObject timerConnection.disconnect(); timer->setParent(0); - timer->deleteLater(); + DeferCall::deleteLater(timer); retryTimerConnection.disconnect(); retryTimer->setParent(0); - retryTimer->deleteLater(); + DeferCall::deleteLater(retryTimer); } void start() @@ -259,7 +279,7 @@ class HttpSession::Private : public QObject assert(state == NotStarted); // set up implicit channels - QPointer self = this; + std::weak_ptr self = q->d; foreach(const QString &name, adata.implicitChannels) { if(!channels.contains(name)) @@ -271,7 +291,7 @@ class HttpSession::Private : public QObject subscribeCallback.call({q, name}); - assert(self); // deleting here would leak subscriptions/connections + assert(!self.expired()); // deleting here would leak subscriptions/connections } } @@ -298,13 +318,13 @@ class HttpSession::Private : public QObject if(!instruct.response.body.isEmpty()) { - // apply ProxyContent filters of all channels + // apply ResponseContent filters of all channels QStringList allFilters; foreach(const Instruct::Channel &c, instruct.channels) { foreach(const QString &filter, c.filters) { - if((Filter::targets(filter) & Filter::ProxyContent) && !allFilters.contains(filter)) + if((Filter::targets(filter) & Filter::ResponseContent) && !allFilters.contains(filter)) allFilters += filter; } } @@ -411,97 +431,21 @@ class HttpSession::Private : public QObject if(f.type == PublishFormat::HttpResponse) { - if(state != Holding) - return; - assert(instruct.holdMode == Instruct::ResponseHold); - if(!channels.contains(item.channel)) - { - log_debug("httpsession: received publish for channel with no subscription, dropping"); - return; - } - - Instruct::Channel &channel = channels[item.channel]; - - if(!channel.prevId.isNull()) - { - if(channel.prevId != item.prevId) - { - log_debug("last ID inconsistency (got=%s, expected=%s), retrying", qPrintable(item.prevId), qPrintable(channel.prevId)); - publishLastIds->remove(item.channel); - - update(LowPriority); - return; - } - - channel.prevId = item.id; - } - - if(f.haveContentFilters) + if(state == SendingQueue || state == Holding) { - // ensure content filters match - QStringList contentFilters; - foreach(const QString &f, channels[item.channel].filters) - { - if(Filter::targets(f) & Filter::MessageContent) - contentFilters += f; - } - if(contentFilters != f.contentFilters) + if(publishQueue.count() < PUBLISH_QUEUE_MAX) { - errorMessage = QString("content filter mismatch: subscription=%1 message=%2").arg(contentFilters.join(","), f.contentFilters.join(",")); - doError(); - return; - } - } - - QHash prevIds; - QHashIterator it(channels); - while(it.hasNext()) - { - it.next(); - const Instruct::Channel &c = it.value(); - prevIds[c.name] = c.prevId; - } - - Filter::Context fc; - fc.prevIds = prevIds; - fc.subscriptionMeta = instruct.meta; - fc.publishMeta = item.meta; + publishQueue += QueuedItem(item, exposeHeaders); - FilterStack fs(fc, channels[item.channel].filters); - - if(fs.sendAction() == Filter::Drop) - return; - - // NOTE: http-response mode doesn't support a close - // action since it's better to send a real response - - if(f.action == PublishFormat::Send) - { - QByteArray body; - if(f.haveBodyPatch) - { - body = applyBodyPatch(instruct.response.body, f.bodyPatch); + if(state == Holding) + sendQueue(); } else { - body = f.body; - } - - body = fs.process(body); - if(body.isNull()) - { - errorMessage = QString("filter error: %1").arg(fs.errorMessage()); - doError(); - return; + log_debug("httpsession: publish queue at max, dropping"); } - - respond(f.code, f.reason, f.headers, body, exposeHeaders); - } - else if(f.action == PublishFormat::Hint) - { - update(HighPriority); } } else if(f.type == PublishFormat::HttpStream) @@ -510,10 +454,10 @@ class HttpSession::Private : public QObject { if(publishQueue.count() < PUBLISH_QUEUE_MAX) { - publishQueue += item; + publishQueue += QueuedItem(item); if(state == Holding) - trySendQueue(); + sendQueue(); } else { @@ -710,20 +654,20 @@ class HttpSession::Private : public QObject } } - QPointer self = this; + std::weak_ptr self = q->d; foreach(const QString &channel, channelsRemoved) { unsubscribeCallback.call({q, channel}); - assert(self); // deleting here would leak subscriptions/connections + assert(!self.expired()); // deleting here would leak subscriptions/connections } foreach(const QString &channel, channelsAdded) { subscribeCallback.call({q, channel}); - assert(self); // deleting here would leak subscriptions/connections + assert(!self.expired()); // deleting here would leak subscriptions/connections } if(instruct.holdMode == Instruct::ResponseHold) @@ -778,7 +722,8 @@ class HttpSession::Private : public QObject // drop any non-matching queued items while(!publishQueue.isEmpty()) { - PublishItem &item = publishQueue.first(); + const QueuedItem &qi = publishQueue.first(); + const PublishItem &item = qi.item; if(!channels.contains(item.channel)) { @@ -799,29 +744,33 @@ class HttpSession::Private : public QObject break; } - if(!publishQueue.isEmpty()) - { - state = SendingQueue; - trySendQueue(); - } - else - { - sendQueueDone(); - } + // if there are items to send, this will send them. if there are + // no items to send, this will end up changing state to Holding + sendQueue(); } } - void trySendQueue() + void sendQueue() { - assert(instruct.holdMode == Instruct::StreamHold); + state = SendingQueue; - while(!publishQueue.isEmpty() && req->writeBytesAvailable() > 0) + processPublishQueue(); + } + + void processPublishQueue() + { + assert(!inProcessPublishQueue); + inProcessPublishQueue = true; + + while(state == SendingQueue && !publishQueue.isEmpty() && req->writeBytesAvailable() > 0 && !messageFilters) { - PublishItem item = publishQueue.takeFirst(); + const QueuedItem &qi = publishQueue.first(); + const PublishItem &item = qi.item; if(!channels.contains(item.channel)) { log_debug("httpsession: received publish for channel with no subscription, dropping"); + publishQueue.removeFirst(); continue; } @@ -839,11 +788,9 @@ class HttpSession::Private : public QObject update(LowPriority); break; } - - channel.prevId = item.id; } - PublishFormat &f = item.format; + const PublishFormat &f = item.format; if(f.haveContentFilters) { @@ -856,12 +803,28 @@ class HttpSession::Private : public QObject } if(contentFilters != f.contentFilters) { - errorMessage = QString("content filter mismatch: subscription=%1 message=%2").arg(contentFilters.join(","), f.contentFilters.join(",")); - doError(); - break; + publishQueue.removeFirst(); + + if(adata.debug) + { + errorMessage = QString("content filter mismatch: subscription=%1 message=%2").arg(contentFilters.join(","), f.contentFilters.join(",")); + doError(); + break; + } + + continue; } } + QByteArray body; + if(f.type == PublishFormat::HttpResponse && f.haveBodyPatch) + body = applyBodyPatch(instruct.response.body, f.bodyPatch); + else + body = f.body; + + messageFilters = std::make_unique(channels[item.channel].filters); + messageFiltersFinishedConnection = messageFilters->finished.connect(boost::bind(&Private::messageFiltersFinished, this, boost::placeholders::_1)); + QHash prevIds; QHashIterator it(channels); while(it.hasNext()) @@ -875,82 +838,66 @@ class HttpSession::Private : public QObject fc.prevIds = prevIds; fc.subscriptionMeta = instruct.meta; fc.publishMeta = item.meta; + fc.zhttpOut = outZhttp; + fc.currentUri = currentUri; + fc.route = adata.route; + fc.trusted = adata.trusted; + fc.limiter = filterLimiter; + + // may call messageFiltersFinished immediately. if it does, queue + // processing will continue. else, the loop will end and queue + // processing will resume after the filters finish + messageFilters->start(fc, body); + } - FilterStack fs(fc, channels[item.channel].filters); - - if(fs.sendAction() == Filter::Drop) - continue; + if(!messageFilters) + { + // the state changed, the queue is empty, or the client buffer is full - if(f.action == PublishFormat::Send) + if(state != SendingQueue || publishQueue.isEmpty()) { - QByteArray body = fs.process(f.body); - if(body.isNull()) - { - errorMessage = QString("filter error: %1").arg(fs.errorMessage()); - doError(); - break; - } - - writeBody(body); - - // restart keep alive timer - adjustKeepAlive(); - - if(!nextUri.isEmpty() && instruct.nextLinkTimeout >= 0) - { - activeChannels += item.channel; - if(activeChannels.count() == channels.count()) - { - activeChannels.clear(); - - updateManager->registerSession(q, instruct.nextLinkTimeout, nextUri); - } - } + // if the state changed or the queue is empty then we're done + sendQueueDone(); } - else if(f.action == PublishFormat::Hint) + else { - // clear queue since any items will be redundant - publishQueue.clear(); + // client buffer can only be full in stream mode + assert(instruct.holdMode == Instruct::StreamHold); - update(HighPriority); - break; - } - else if(f.action == PublishFormat::Close) - { - prepareToClose(); - req->endBody(); - break; - } - } + // NOTE: we can end up here multiple times in a single pass + // of the queue if the client buffer becomes full multiple + // times. so, whatever happens here should be idempotent and + // cheap. - if(state == SendingQueue) - { - if(publishQueue.isEmpty()) - sendQueueDone(); - } - else if(state == Holding) - { - if(!publishQueue.isEmpty()) - { - // if backlogged, turn off timers until we're able to send again + // turn off timers until we're able to send again timer->stop(); updateManager->unregisterSession(q); } } + + inProcessPublishQueue = false; } void sendQueueDone() { + // if the state changed during queue processing (e.g. to Closing), + // then we want to leave the state alone and do nothing else + if(state != SendingQueue) + return; + state = Holding; - activeChannels.clear(); + if(instruct.holdMode == Instruct::StreamHold) + { + activeChannels.clear(); - // start keep alive timer, if it wasn't started already - if(!timer->isActive()) - setupKeepAlive(); + // start keep alive timer, if it wasn't started already + if(!timer->isActive()) + setupKeepAlive(); - if(!nextUri.isEmpty() && instruct.nextLinkTimeout >= 0) - updateManager->registerSession(q, instruct.nextLinkTimeout, nextUri); + if(!nextUri.isEmpty() && instruct.nextLinkTimeout >= 0) + updateManager->registerSession(q, instruct.nextLinkTimeout, nextUri); + } if(needUpdate) update(needUpdatePriority); @@ -1088,7 +1035,7 @@ class HttpSession::Private : public QObject ZhttpRequest::Rid rid = req->rid(); QByteArray cid = rid.first + ':' + rid.second; - QPointer self = this; + std::weak_ptr self = q->d; QHashIterator it(channels); while(it.hasNext()) @@ -1098,7 +1045,7 @@ class HttpSession::Private : public QObject unsubscribeCallback.call({q, channel}); - assert(self); // deleting here would leak subscriptions/connections + assert(!self.expired()); // deleting here would leak subscriptions/connections } if(retry) @@ -1229,7 +1176,7 @@ class HttpSession::Private : public QObject passthroughData["prefer-internal"] = true; } - // these fields are needed in case proxy routing is not used + // needed in case internal routing is not used if(adata.trusted) passthroughData["trusted"] = true; @@ -1246,7 +1193,7 @@ class HttpSession::Private : public QObject if(!outZhttp) { errorMessage = "Instruct contained link, but handler not configured for outbound requests."; - QMetaObject::invokeMethod(this, "doError", Qt::QueuedConnection); + deferCall.defer([=] { doError(); }); return; } @@ -1459,7 +1406,97 @@ class HttpSession::Private : public QObject req->writeBody(body); } -private slots: + void messageFiltersFinished(const Filter::MessageFilter::Result &result) + { + QueuedItem qi = publishQueue.takeFirst(); + + messageFiltersFinishedConnection.disconnect(); + messageFilters.reset(); + + if(!result.errorMessage.isNull()) + { + if(adata.debug) + { + errorMessage = QString("filter error: %1").arg(result.errorMessage); + doError(); + return; + } + } + else + { + processItem(qi.item, result.sendAction, result.content, qi.exposeHeaders); + } + + // if filters finished asynchronously then we need to resume processing + if(!inProcessPublishQueue) + processPublishQueue(); + } + + void processItem(const PublishItem &item, Filter::SendAction sendAction, const QByteArray &content, const QList &exposeHeaders) + { + const PublishFormat &f = item.format; + + Instruct::Channel &channel = channels[item.channel]; + + if(!channel.prevId.isNull()) + channel.prevId = item.id; + + if(instruct.holdMode == Instruct::ResponseHold) + { + if(sendAction == Filter::Drop) + return; + + // NOTE: http-response mode doesn't support a close + // action since it's better to send a real response + + if(f.action == PublishFormat::Send) + { + respond(f.code, f.reason, f.headers, content, exposeHeaders); + } + else if(f.action == PublishFormat::Hint) + { + update(HighPriority); + } + } + else if(instruct.holdMode == Instruct::StreamHold) + { + if(sendAction == Filter::Drop) + return; + + if(f.action == PublishFormat::Send) + { + writeBody(content); + + // restart keep alive timer + adjustKeepAlive(); + + if(!nextUri.isEmpty() && instruct.nextLinkTimeout >= 0) + { + activeChannels += item.channel; + if(activeChannels.count() == channels.count()) + { + activeChannels.clear(); + + // all channels had activity. reset the timeout + updateManager->registerSession(q, instruct.nextLinkTimeout, nextUri, true); + } + } + } + else if(f.action == PublishFormat::Hint) + { + // clear queue since any items will be redundant + publishQueue.clear(); + + update(HighPriority); + } + else if(f.action == PublishFormat::Close) + { + prepareToClose(); + req->endBody(); + } + } + } + void doError() { if(instruct.holdMode == Instruct::ResponseHold) @@ -1507,9 +1544,13 @@ private slots: { tryProcessOutReq(); } - else if(state == SendingQueue || state == Holding) + else if(state == SendingQueue) { - trySendQueue(); + // in this state, the writeBytesChanged signal is only + // interesting if it indicates write bytes are available + + if(req->writeBytesAvailable() > 0) + processPublishQueue(); } } @@ -1558,13 +1599,13 @@ private slots: // won't be used for anything else instruct = i; - // apply ProxyContent filters of all channels + // apply ResponseContent filters of all channels QStringList allFilters; foreach(const Instruct::Channel &c, instruct.channels) { foreach(const QString &filter, c.filters) { - if((Filter::targets(filter) & Filter::ProxyContent) && !allFilters.contains(filter)) + if((Filter::targets(filter) & Filter::ResponseContent) && !allFilters.contains(filter)) allFilters += filter; } } @@ -1623,7 +1664,6 @@ private slots: } } -private: void timer_timeout() { if(instruct.holdMode == Instruct::ResponseHold) @@ -1647,16 +1687,13 @@ private slots: } }; -HttpSession::HttpSession(ZhttpRequest *req, const HttpSession::AcceptData &adata, const Instruct &instruct, ZhttpManager *zhttpOut, StatsManager *stats, RateLimiter *updateLimiter, PublishLastIds *publishLastIds, HttpSessionUpdateManager *updateManager, int connectionSubscriptionMax, QObject *parent) : +HttpSession::HttpSession(ZhttpRequest *req, const HttpSession::AcceptData &adata, const Instruct &instruct, ZhttpManager *zhttpOut, StatsManager *stats, RateLimiter *updateLimiter, const std::shared_ptr &filterLimiter, PublishLastIds *publishLastIds, HttpSessionUpdateManager *updateManager, int connectionSubscriptionMax, QObject *parent) : QObject(parent) { - d = new Private(this, req, adata, instruct, zhttpOut, stats, updateLimiter, publishLastIds, updateManager, connectionSubscriptionMax); + d = std::make_shared(this, req, adata, instruct, zhttpOut, stats, updateLimiter, filterLimiter, publishLastIds, updateManager, connectionSubscriptionMax); } -HttpSession::~HttpSession() -{ - delete d; -} +HttpSession::~HttpSession() = default; Instruct::HoldMode HttpSession::holdMode() const { diff --git a/src/handler/httpsession.h b/src/handler/httpsession.h index 468ce0251..bff78dd5c 100644 --- a/src/handler/httpsession.h +++ b/src/handler/httpsession.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2016-2023 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -31,11 +31,19 @@ #include "inspectdata.h" #include "zhttprequest.h" #include "instruct.h" +#include "filter.h" #include +// each session can have a bunch of timers: +// incoming request +// outgoing request +// 2 additional timers +// filter timers +// a few more just in case +#define TIMERS_PER_HTTPSESSION ((TIMERS_PER_ZHTTPREQUEST * 2) + 2 + TIMERS_PER_MESSAGEFILTERSTACK + 4) + using Connection = boost::signals2::scoped_connection; -class QTimer; class ZhttpManager; class StatsManager; class PublishItem; @@ -88,7 +96,7 @@ class HttpSession : public QObject } }; - HttpSession(ZhttpRequest *req, const HttpSession::AcceptData &adata, const Instruct &instruct, ZhttpManager *outZhttp, StatsManager *stats, RateLimiter *updateLimiter, PublishLastIds *publishLastIds, HttpSessionUpdateManager *updateManager, int connectionSubscriptionMax, QObject *parent = 0); + HttpSession(ZhttpRequest *req, const HttpSession::AcceptData &adata, const Instruct &instruct, ZhttpManager *outZhttp, StatsManager *stats, RateLimiter *updateLimiter, const std::shared_ptr &filterLimiter, PublishLastIds *publishLastIds, HttpSessionUpdateManager *updateManager, int connectionSubscriptionMax, QObject *parent = 0); ~HttpSession(); Instruct::HoldMode holdMode() const; @@ -114,7 +122,7 @@ class HttpSession : public QObject private: class Private; friend class Private; - Private *d; + std::shared_ptr d; }; #endif diff --git a/src/handler/httpsessionupdatemanager.cpp b/src/handler/httpsessionupdatemanager.cpp index 1f403f07b..76b950860 100644 --- a/src/handler/httpsessionupdatemanager.cpp +++ b/src/handler/httpsessionupdatemanager.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2016 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -22,8 +23,9 @@ #include "httpsessionupdatemanager.h" -#include #include +#include "timer.h" +#include "defercall.h" #include "httpsession.h" class HttpSessionUpdateManager::Private : public QObject @@ -37,12 +39,12 @@ class HttpSessionUpdateManager::Private : public QObject QPair key; QSet sessions; QSet deferredSessions; - QTimer *timer; + Timer *timer; }; HttpSessionUpdateManager *q; QHash, Bucket*> buckets; - QHash bucketsByTimer; + QHash bucketsByTimer; QHash bucketsBySession; Private(HttpSessionUpdateManager *_q) : @@ -61,7 +63,7 @@ class HttpSessionUpdateManager::Private : public QObject bucket->timer->disconnect(this); bucket->timer->setParent(0); - bucket->timer->deleteLater(); + DeferCall::deleteLater(bucket->timer); delete bucket; } } @@ -76,11 +78,11 @@ class HttpSessionUpdateManager::Private : public QObject bucket->timer->disconnect(this); bucket->timer->setParent(0); - bucket->timer->deleteLater(); + DeferCall::deleteLater(bucket->timer); delete bucket; } - void registerSession(HttpSession *hs, int timeout, const QUrl &uri) + void registerSession(HttpSession *hs, int timeout, const QUrl &uri, bool resetTimeout) { QUrl tmp = uri; tmp.setQuery(QString()); // remove the query part @@ -91,9 +93,11 @@ class HttpSessionUpdateManager::Private : public QObject { if(bucket->sessions.contains(hs)) { - // if the session is already in this bucket, flag it - // for later processing - bucket->deferredSessions += hs; + if(resetTimeout) + { + // flag for later processing + bucket->deferredSessions += hs; + } } else { @@ -112,10 +116,8 @@ class HttpSessionUpdateManager::Private : public QObject bucket = new Bucket; bucket->key = key; bucket->sessions += hs; - bucket->timer = new QTimer(this); - QObject::connect(bucket->timer, &QTimer::timeout, [this, timer=bucket->timer]() { - this->timer_timeout(timer); - }); + bucket->timer = new Timer; + bucket->timer->timeout.connect(boost::bind(&Private::timer_timeout, this, bucket->timer)); buckets[key] = bucket; bucketsByTimer[bucket->timer] = bucket; @@ -140,7 +142,7 @@ class HttpSessionUpdateManager::Private : public QObject } private: - void timer_timeout(QTimer *timer) + void timer_timeout(Timer *timer) { Bucket *bucket = bucketsByTimer.value(timer); if(!bucket) @@ -185,9 +187,9 @@ HttpSessionUpdateManager::~HttpSessionUpdateManager() delete d; } -void HttpSessionUpdateManager::registerSession(HttpSession *hs, int timeout, const QUrl &uri) +void HttpSessionUpdateManager::registerSession(HttpSession *hs, int timeout, const QUrl &uri, bool resetTimeout) { - d->registerSession(hs, timeout, uri); + d->registerSession(hs, timeout, uri, resetTimeout); } void HttpSessionUpdateManager::unregisterSession(HttpSession *hs) diff --git a/src/handler/httpsessionupdatemanager.h b/src/handler/httpsessionupdatemanager.h index 1a491ebd4..e22e464a0 100644 --- a/src/handler/httpsessionupdatemanager.h +++ b/src/handler/httpsessionupdatemanager.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2016 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -25,6 +26,8 @@ #include +#define TIMERS_PER_UNIQUE_UPDATE_REGISTRATION 1 + class QUrl; class HttpSession; @@ -34,7 +37,9 @@ class HttpSessionUpdateManager : public QObject HttpSessionUpdateManager(QObject *parent = 0); ~HttpSessionUpdateManager(); - void registerSession(HttpSession *hs, int timeout, const QUrl &uri); + // no-op if session already registered and resetTimeout=false + void registerSession(HttpSession *hs, int timeout, const QUrl &uri, bool resetTimeout = false); + void unregisterSession(HttpSession *hs); private: diff --git a/src/handler/instruct.cpp b/src/handler/instruct.cpp index 5b2c5c0d7..4f369483f 100644 --- a/src/handler/instruct.cpp +++ b/src/handler/instruct.cpp @@ -29,6 +29,7 @@ #include "qtcompat.h" #include "variantutil.h" #include "statusreasons.h" +#include "filter.h" #define DEFAULT_RESPONSE_TIMEOUT 55 #define MINIMUM_RESPONSE_TIMEOUT 5 @@ -147,6 +148,12 @@ Instruct Instruct::fromResponse(const HttpResponseData &response, bool *ok, QStr c.filters += QString::fromUtf8(param.second); } + if(c.filters.count() > MESSAGEFILTERSTACK_SIZE_MAX) + { + setError(ok, errorMessage, QString("too many filters for channel '%1'").arg(c.name)); + return Instruct(); + } + channels += c; } diff --git a/src/handler/mod.rs b/src/handler/mod.rs index fcd599928..50f900e19 100644 --- a/src/handler/mod.rs +++ b/src/handler/mod.rs @@ -20,6 +20,11 @@ mod tests { use crate::ffi; use std::ffi::OsStr; + fn filter_test(args: &[&OsStr]) -> u8 { + // SAFETY: safe to call + unsafe { call_c_main(ffi::filter_test, args) as u8 } + } + fn jsonpatch_test(args: &[&OsStr]) -> u8 { // SAFETY: safe to call unsafe { call_c_main(ffi::jsonpatch_test, args) as u8 } @@ -50,6 +55,11 @@ mod tests { unsafe { call_c_main(ffi::handlerengine_test, args) as u8 } } + #[test] + fn filter() { + assert!(qtest::run(filter_test)); + } + #[test] fn jsonpatch() { assert!(qtest::run(jsonpatch_test)); diff --git a/src/handler/ratelimiter.cpp b/src/handler/ratelimiter.cpp index fcf348aa8..ea8e99153 100644 --- a/src/handler/ratelimiter.cpp +++ b/src/handler/ratelimiter.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2016 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -24,8 +25,8 @@ #include #include -#include -#include +#include "timer.h" +#include "defercall.h" #define MIN_BATCH_INTERVAL 25 @@ -67,19 +68,21 @@ class RateLimiter::Private : public QObject } }; + RateLimiter *q; int rate; int hwm; bool batchWaitEnabled; QMap buckets; QString lastKey; - QTimer *timer; + Timer *timer; bool firstPass; int batchInterval; int batchSize; bool lastBatchEmpty; - Private(QObject *_q) : + Private(RateLimiter *_q) : QObject(_q), + q(_q), rate(-1), hwm(-1), batchWaitEnabled(false), @@ -87,15 +90,15 @@ class RateLimiter::Private : public QObject batchSize(-1), lastBatchEmpty(false) { - timer = new QTimer(this); - connect(timer, &QTimer::timeout, this, &Private::timeout); + timer = new Timer; + timer->timeout.connect(boost::bind(&Private::timeout, this)); } ~Private() { timer->disconnect(this); timer->setParent(0); - timer->deleteLater(); + DeferCall::deleteLater(timer); } void setRate(int actionsPerSecond) @@ -226,7 +229,7 @@ class RateLimiter::Private : public QObject it = buckets.begin(); } - QPointer self = this; + std::weak_ptr self = q->d; int processed = 0; while((batchSize < 1 || processed < batchSize) && it != buckets.end()) @@ -246,7 +249,7 @@ class RateLimiter::Private : public QObject bool ret = action->execute(); delete action; - if(!self) + if(self.expired()) return false; if(ret) @@ -287,7 +290,6 @@ class RateLimiter::Private : public QObject return true; } -private slots: void timeout() { if(!processBatch()) @@ -301,7 +303,7 @@ private slots: RateLimiter::RateLimiter() { - d = std::make_unique(this); + d = std::make_shared(this); } RateLimiter::~RateLimiter() = default; diff --git a/src/handler/ratelimiter.h b/src/handler/ratelimiter.h index 77fb344d9..5afd41f7f 100644 --- a/src/handler/ratelimiter.h +++ b/src/handler/ratelimiter.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2016 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -51,7 +52,7 @@ class RateLimiter : public QObject private: class Private; - std::unique_ptr d; + std::shared_ptr d; }; #endif diff --git a/src/handler/refreshworker.h b/src/handler/refreshworker.h index a97797088..37f0fc4c3 100644 --- a/src/handler/refreshworker.h +++ b/src/handler/refreshworker.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2017-2020 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -50,8 +51,6 @@ class RefreshWorker : public Deferred void refreshNextCid(); void respondError(const QByteArray &condition); - -private slots: void proxyRefresh_finished(const DeferredResult &result); }; diff --git a/src/handler/sequencer.cpp b/src/handler/sequencer.cpp index a5dee9ba9..d4b31ba89 100644 --- a/src/handler/sequencer.cpp +++ b/src/handler/sequencer.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2016-2021 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -23,8 +24,9 @@ #include "sequencer.h" #include -#include #include "log.h" +#include "timer.h" +#include "defercall.h" #include "publishitem.h" #include "publishlastids.h" @@ -67,7 +69,7 @@ class Sequencer::Private : public QObject PublishLastIds *lastIds; QHash pendingItemsByChannel; QMap, PendingItem*> pendingItemsByTime; - QTimer *expireTimer; + Timer *expireTimer; int pendingExpireMSecs; int idCacheTtl; QHash, CachedId*> idCacheById; @@ -80,15 +82,15 @@ class Sequencer::Private : public QObject pendingExpireMSecs(DEFAULT_PENDING_EXPIRE), idCacheTtl(-1) { - expireTimer = new QTimer(this); - connect(expireTimer, &QTimer::timeout, this, &Private::expireTimer_timeout); + expireTimer = new Timer; + expireTimer->timeout.connect(boost::bind(&Private::expireTimer_timeout, this)); } ~Private() { expireTimer->disconnect(this); expireTimer->setParent(0); - expireTimer->deleteLater(); + DeferCall::deleteLater(expireTimer); qDeleteAll(idCacheById); } @@ -229,7 +231,6 @@ class Sequencer::Private : public QObject } } -private slots: void expireTimer_timeout() { qint64 now = QDateTime::currentMSecsSinceEpoch(); diff --git a/src/handler/tests.pri b/src/handler/tests.pri index 412617e65..b4ade3cbe 100644 --- a/src/handler/tests.pri +++ b/src/handler/tests.pri @@ -2,6 +2,7 @@ INCLUDES += \ $$PWD/handlertests.h SOURCES += \ + $$PWD/filtertest.cpp \ $$PWD/jsonpatchtest.cpp \ $$PWD/instructtest.cpp \ $$PWD/idformattest.cpp \ diff --git a/src/handler/wssession.cpp b/src/handler/wssession.cpp index fc646596c..b113ab0ec 100644 --- a/src/handler/wssession.cpp +++ b/src/handler/wssession.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2020 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -23,43 +23,52 @@ #include "wssession.h" -#include #include #include "log.h" +#include "timer.h" +#include "defercall.h" +#include "filter.h" +#include "publishitem.h" +#include "publishformat.h" #define WSCONTROL_REQUEST_TIMEOUT 8000 WsSession::WsSession(QObject *parent) : QObject(parent), nextReqId(0), - logLevel(LOG_LEVEL_DEBUG) + debug(false), + logLevel(LOG_LEVEL_DEBUG), + targetTrusted(false), + ttl(0), + inProcessPublishQueue(false), + closed(false) { - expireTimer = new QTimer(this); + expireTimer = new Timer; expireTimer->setSingleShot(true); - connect(expireTimer, &QTimer::timeout, this, &WsSession::expireTimer_timeout); + expireTimer->timeout.connect(boost::bind(&WsSession::expireTimer_timeout, this)); - delayedTimer = new QTimer(this); + delayedTimer = new Timer; delayedTimer->setSingleShot(true); - connect(delayedTimer, &QTimer::timeout, this, &WsSession::delayedTimer_timeout); + delayedTimer->timeout.connect(boost::bind(&WsSession::delayedTimer_timeout, this)); - requestTimer = new QTimer(this); + requestTimer = new Timer; requestTimer->setSingleShot(true); - connect(requestTimer, &QTimer::timeout, this, &WsSession::requestTimer_timeout); + requestTimer->timeout.connect(boost::bind(&WsSession::requestTimer_timeout, this)); } WsSession::~WsSession() { expireTimer->disconnect(this); expireTimer->setParent(0); - expireTimer->deleteLater(); + DeferCall::deleteLater(expireTimer); delayedTimer->disconnect(this); delayedTimer->setParent(0); - delayedTimer->deleteLater(); + DeferCall::deleteLater(delayedTimer); requestTimer->disconnect(this); requestTimer->setParent(0); - requestTimer->deleteLater(); + DeferCall::deleteLater(requestTimer); } void WsSession::refreshExpiration() @@ -94,6 +103,160 @@ void WsSession::ack(int reqId) } } +void WsSession::publish(const PublishItem &item) +{ + const PublishFormat &f = item.format; + + if(f.type != PublishFormat::WebSocketMessage) + return; + + publishQueue += item; + + if(!inProcessPublishQueue) + processPublishQueue(); +} + +void WsSession::processPublishQueue() +{ + assert(!inProcessPublishQueue); + inProcessPublishQueue = true; + + while(!closed && !publishQueue.isEmpty() && !filters) + { + const PublishItem &item = publishQueue.first(); + const PublishFormat &f = item.format; + + if(f.haveContentFilters) + { + // ensure content filters match + QStringList contentFilters; + foreach(const QString &f, channelFilters[item.channel]) + { + if(Filter::targets(f) & Filter::MessageContent) + contentFilters += f; + } + if(contentFilters != f.contentFilters) + { + publishQueue.removeFirst(); + + if(debug) + { + QString errorMessage = QString("content filter mismatch: subscription=%1 message=%2").arg(contentFilters.join(","), f.contentFilters.join(",")); + sendCloseError(errorMessage); + break; + } + + continue; + } + } + + filters = std::make_unique(channelFilters[item.channel]); + filtersFinishedConnection = filters->finished.connect(boost::bind(&WsSession::filtersFinished, this, boost::placeholders::_1)); + + Filter::Context fc; + fc.subscriptionMeta = meta; + fc.publishMeta = item.meta; + fc.zhttpOut = zhttpOut; + fc.currentUri = requestData.uri; + fc.route = route; + fc.trusted = targetTrusted; + fc.limiter = filterLimiter; + + // may call filtersFinished immediately. if it does, queue processing + // will continue. else, the loop will end and queue processing will + // resume after the filters finish + filters->start(fc, f.body); + } + + inProcessPublishQueue = false; +} + +void WsSession::filtersFinished(const Filter::MessageFilter::Result &result) +{ + PublishItem item = publishQueue.takeFirst(); + + filtersFinishedConnection.disconnect(); + filters.reset(); + + if(!result.errorMessage.isNull()) + { + if(debug) + { + QString errorMessage = QString("filter error: %1").arg(result.errorMessage); + sendCloseError(errorMessage); + return; + } + } + else + { + afterFilters(item, result.sendAction, result.content); + } + + // if filters finished asynchronously then we need to resume processing + if(!inProcessPublishQueue) + processPublishQueue(); +} + +void WsSession::afterFilters(const PublishItem &item, Filter::SendAction sendAction, const QByteArray &content) +{ + if(sendAction == Filter::Drop) + return; + + const PublishFormat &f = item.format; + + // TODO: hint support for websockets? + if(f.action != PublishFormat::Send && f.action != PublishFormat::Close && f.action != PublishFormat::Refresh) + return; + + WsControlPacket::Item i; + i.cid = cid.toUtf8(); + + if(f.action == PublishFormat::Send) + { + i.type = WsControlPacket::Item::Send; + + switch(f.messageType) + { + case PublishFormat::Text: i.contentType = "text"; break; + case PublishFormat::Binary: i.contentType = "binary"; break; + case PublishFormat::Ping: i.contentType = "ping"; break; + case PublishFormat::Pong: i.contentType = "pong"; break; + default: return; // unrecognized type, skip + } + + i.message = content; + } + else if(f.action == PublishFormat::Close) + { + closed = true; + + i.type = WsControlPacket::Item::Close; + i.code = f.code; + i.reason = f.reason; + } + else if(f.action == PublishFormat::Refresh) + { + i.type = WsControlPacket::Item::Refresh; + } + + send(i); +} + +void WsSession::sendCloseError(const QString &message) +{ + closed = true; + + WsControlPacket::Item i; + i.cid = cid.toUtf8(); + i.type = WsControlPacket::Item::Close; + i.code = 1011; + + if(debug) + i.reason = message.toUtf8(); + + send(i); +} + void WsSession::setupRequestTimer() { if(!pendingRequests.isEmpty()) @@ -137,7 +300,15 @@ void WsSession::delayedTimer_timeout() pendingRequests[reqId] = QDateTime::currentMSecsSinceEpoch() + WSCONTROL_REQUEST_TIMEOUT; setupRequestTimer(); - send(reqId, delayedType, message); + WsControlPacket::Item i; + i.cid = cid.toUtf8(); + i.requestId = QByteArray::number(reqId); + i.type = WsControlPacket::Item::Send; + i.contentType = delayedType; + i.message = message; + i.queue = true; + + send(i); } void WsSession::requestTimer_timeout() diff --git a/src/handler/wssession.h b/src/handler/wssession.h index efd684fd4..5e3a8abed 100644 --- a/src/handler/wssession.h +++ b/src/handler/wssession.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2020 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -28,12 +28,22 @@ #include #include #include "packet/httprequestdata.h" +#include "packet/wscontrolpacket.h" +#include "ratelimiter.h" +#include "filter.h" #include +// each session can have a bunch of timers: +// 3 misc timers +// filter timers +#define TIMERS_PER_WSSESSION (3 + TIMERS_PER_MESSAGEFILTERSTACK) + using Signal = boost::signals2::signal; using Connection = boost::signals2::scoped_connection; -class QTimer; +class Timer; +class ZhttpManager; +class PublishItem; class WsSession : public QObject { @@ -43,11 +53,13 @@ class WsSession : public QObject QByteArray peer; QString cid; int nextReqId; + bool debug; QString channelPrefix; int logLevel; HttpRequestData requestData; QString route; QString statsRoute; + bool targetTrusted; QString sid; QHash meta; QHash channelFilters; // k=channel, v=list(filters) @@ -59,9 +71,16 @@ class WsSession : public QObject QByteArray delayedType; QByteArray delayedMessage; QHash pendingRequests; - QTimer *expireTimer; - QTimer *delayedTimer; - QTimer *requestTimer; + Timer *expireTimer; + Timer *delayedTimer; + Timer *requestTimer; + QList publishQueue; + ZhttpManager *zhttpOut; + std::shared_ptr filterLimiter; + std::unique_ptr filters; + Connection filtersFinishedConnection; + bool inProcessPublishQueue; + bool closed; WsSession(QObject *parent = 0); ~WsSession(); @@ -70,15 +89,18 @@ class WsSession : public QObject void flushDelayed(); void sendDelayed(const QByteArray &type, const QByteArray &message, int timeout); void ack(int reqId); + void publish(const PublishItem &item); + void sendCloseError(const QString &message); - boost::signals2::signal send; + boost::signals2::signal send; Signal expired; Signal error; private: + void processPublishQueue(); + void filtersFinished(const Filter::MessageFilter::Result &result); + void afterFilters(const PublishItem &item, Filter::SendAction sendAction, const QByteArray &content); void setupRequestTimer(); - -private slots: void expireTimer_timeout(); void delayedTimer_timeout(); void requestTimer_timeout(); diff --git a/src/lib.rs b/src/lib.rs index 9613d0a93..e70048413 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,6 +37,7 @@ pub mod runner; macro_rules! import_cpp { ($($tt:tt)*) => { #[link(name = "pushpin-cpp")] + #[link(name = "hiredis")] #[cfg_attr( all(target_os = "macos", qt_lib_prefix = "Qt"), link(name = "QtCore", kind = "framework"), @@ -80,6 +81,7 @@ macro_rules! import_cpptest { ($($tt:tt)*) => { #[link(name = "pushpin-cpptest")] #[link(name = "pushpin-cpp")] + #[link(name = "hiredis")] #[cfg_attr( all(target_os = "macos", qt_lib_prefix = "Qt"), link(name = "QtCore", kind = "framework"), @@ -129,8 +131,10 @@ pub mod ffi { import_cpptest! { pub fn httpheaders_test(argc: libc::c_int, argv: *const *const libc::c_char) -> libc::c_int; pub fn jwt_test(argc: libc::c_int, argv: *const *const libc::c_char) -> libc::c_int; + pub fn eventloop_test(argc: libc::c_int, argv: *const *const libc::c_char) -> libc::c_int; pub fn routesfile_test(argc: libc::c_int, argv: *const *const libc::c_char) -> libc::c_int; pub fn proxyengine_test(argc: libc::c_int, argv: *const *const libc::c_char) -> libc::c_int; + pub fn filter_test(argc: libc::c_int, argv: *const *const libc::c_char) -> libc::c_int; pub fn jsonpatch_test(argc: libc::c_int, argv: *const *const libc::c_char) -> libc::c_int; pub fn instruct_test(argc: libc::c_int, argv: *const *const libc::c_char) -> libc::c_int; pub fn idformat_test(argc: libc::c_int, argv: *const *const libc::c_char) -> libc::c_int; diff --git a/src/m2adapter/m2adapterapp.cpp b/src/m2adapter/m2adapterapp.cpp index 0ae2952d1..e372cfb72 100644 --- a/src/m2adapter/m2adapterapp.cpp +++ b/src/m2adapter/m2adapterapp.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2013-2022 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -418,17 +418,17 @@ class M2AdapterApp::Private : public QObject ArgsData args; QByteArray zhttpInstanceId; QByteArray zwsInstanceId; - QZmq::Socket *m2_in_sock; - QZmq::Socket *m2_out_sock; - QZmq::Socket *zhttp_in_sock; - QZmq::Socket *zhttp_out_sock; - QZmq::Socket *zhttp_out_stream_sock; - QZmq::Socket *zws_in_sock; - QZmq::Socket *zws_out_sock; - QZmq::Socket *zws_out_stream_sock; - QZmq::Valve *m2_in_valve; - QZmq::Valve *zhttp_in_valve; - QZmq::Valve *zws_in_valve; + std::unique_ptr m2_in_sock; + std::unique_ptr m2_out_sock; + std::unique_ptr zhttp_in_sock; + std::unique_ptr zhttp_out_sock; + std::unique_ptr zhttp_out_stream_sock; + std::unique_ptr zws_in_sock; + std::unique_ptr zws_out_sock; + std::unique_ptr zws_out_stream_sock; + std::unique_ptr m2_in_valve; + std::unique_ptr zhttp_in_valve; + std::unique_ptr zws_in_valve; QList m2_send_idents; QHash m2ConnectionsByRid; QHash sessionsByM2Rid; @@ -462,17 +462,6 @@ class M2AdapterApp::Private : public QObject Private(M2AdapterApp *_q) : QObject(_q), q(_q), - m2_in_sock(0), - m2_out_sock(0), - zhttp_in_sock(0), - zhttp_out_sock(0), - zhttp_out_stream_sock(0), - zws_in_sock(0), - zws_out_sock(0), - zws_out_stream_sock(0), - m2_in_valve(0), - zhttp_in_valve(0), - zws_in_valve(0), currentM2RefreshBucket(0), currentSessionRefreshBucket(0), zhttpCancelMeter(0) @@ -494,6 +483,13 @@ class M2AdapterApp::Private : public QObject qDeleteAll(sessionsByZhttpRid); qDeleteAll(sessionsByZwsRid); qDeleteAll(m2ConnectionsByRid); + + for(int n = 0; n < controlPorts.count(); ++n) + { + ControlPort &c = controlPorts[n]; + delete c.sock; + c.sock = 0; + } } void start() @@ -650,7 +646,7 @@ class M2AdapterApp::Private : public QObject zhttpInstanceId = "m2zhttp_" + pidStr; zwsInstanceId = "m2zws_" + pidStr; - m2_in_sock = new QZmq::Socket(QZmq::Socket::Pull, this); + m2_in_sock = std::make_unique(QZmq::Socket::Pull); m2_in_sock->setHwm(DEFAULT_HWM); foreach(const QString &spec, m2_in_specs) { @@ -658,10 +654,10 @@ class M2AdapterApp::Private : public QObject m2_in_sock->connectToAddress(spec); } - m2_in_valve = new QZmq::Valve(m2_in_sock, this); + m2_in_valve = std::make_unique(m2_in_sock.get()); m2InValveConnection = m2_in_valve->readyRead.connect(boost::bind(&Private::m2_in_readyRead, this, boost::placeholders::_1)); - m2_out_sock = new QZmq::Socket(QZmq::Socket::Pub, this); + m2_out_sock = std::make_unique(QZmq::Socket::Pub); m2_out_sock->setShutdownWaitTime(0); m2_out_sock->setHwm(DEFAULT_HWM); m2_out_sock->setWriteQueueEnabled(false); @@ -675,7 +671,7 @@ class M2AdapterApp::Private : public QObject { const QString &spec = m2_control_specs[n]; - QZmq::Socket *sock = new QZmq::Socket(QZmq::Socket::Dealer, this); + QZmq::Socket *sock = new QZmq::Socket(QZmq::Socket::Dealer); sock->setShutdownWaitTime(0); sock->setHwm(1); // queue up 1 outstanding request at most sock->setWriteQueueEnabled(false); @@ -691,7 +687,7 @@ class M2AdapterApp::Private : public QObject if(!zhttp_in_specs.isEmpty()) { - zhttp_in_sock = new QZmq::Socket(QZmq::Socket::Sub, this); + zhttp_in_sock = std::make_unique(QZmq::Socket::Sub); zhttp_in_sock->setHwm(DEFAULT_HWM); zhttp_in_sock->setShutdownWaitTime(0); zhttp_in_sock->subscribe(zhttpInstanceId + ' '); @@ -713,10 +709,10 @@ class M2AdapterApp::Private : public QObject } } - zhttp_in_valve = new QZmq::Valve(zhttp_in_sock, this); + zhttp_in_valve = std::make_unique(zhttp_in_sock.get()); zhttpInValveConnection = zhttp_in_valve->readyRead.connect(boost::bind(&Private::zhttp_in_readyRead, this, boost::placeholders::_1)); - zhttp_out_sock = new QZmq::Socket(QZmq::Socket::Push, this); + zhttp_out_sock = std::make_unique(QZmq::Socket::Push); zhttp_out_sock->setShutdownWaitTime(0); zhttp_out_sock->setHwm(DEFAULT_HWM); if(zhttp_connect) @@ -737,7 +733,7 @@ class M2AdapterApp::Private : public QObject } } - zhttp_out_stream_sock = new QZmq::Socket(QZmq::Socket::Router, this); + zhttp_out_stream_sock = std::make_unique(QZmq::Socket::Router); zhttp_out_stream_sock->setShutdownWaitTime(0); zhttp_out_stream_sock->setHwm(DEFAULT_HWM); if(zhttp_connect) @@ -761,7 +757,7 @@ class M2AdapterApp::Private : public QObject if(!zws_in_specs.isEmpty()) { - zws_in_sock = new QZmq::Socket(QZmq::Socket::Sub, this); + zws_in_sock = std::make_unique(QZmq::Socket::Sub); zws_in_sock->setHwm(DEFAULT_HWM); zws_in_sock->subscribe(zwsInstanceId + ' '); if(zws_connect) @@ -782,10 +778,10 @@ class M2AdapterApp::Private : public QObject } } - zws_in_valve = new QZmq::Valve(zws_in_sock, this); + zws_in_valve = std::make_unique(zws_in_sock.get()); zwsInValveConnection = zws_in_valve->readyRead.connect(boost::bind(&Private::zws_in_readyRead, this, boost::placeholders::_1)); - zws_out_sock = new QZmq::Socket(QZmq::Socket::Push, this); + zws_out_sock = std::make_unique(QZmq::Socket::Push); zws_out_sock->setShutdownWaitTime(0); zws_out_sock->setHwm(DEFAULT_HWM); if(zws_connect) @@ -806,7 +802,7 @@ class M2AdapterApp::Private : public QObject } } - zws_out_stream_sock = new QZmq::Socket(QZmq::Socket::Router, this); + zws_out_stream_sock = std::make_unique(QZmq::Socket::Router); zws_out_stream_sock->setShutdownWaitTime(0); zws_out_stream_sock->setHwm(DEFAULT_HWM); if(zws_connect) diff --git a/src/m2adapter/tools/handoffhandler.py b/src/m2adapter/tools/handoffhandler.py index 1ced7a3f3..87c64859c 100644 --- a/src/m2adapter/tools/handoffhandler.py +++ b/src/m2adapter/tools/handoffhandler.py @@ -4,53 +4,53 @@ import tnetstring import zmq -client_id = 'zhttp-test' +client_id = "zhttp-test" ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc:///tmp/zhttp-test-out') +in_sock.connect("ipc:///tmp/zhttp-test-out") in_stream_sock = ctx.socket(zmq.ROUTER) in_stream_sock.identity = client_id -in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream') +in_stream_sock.connect("ipc:///tmp/zhttp-test-out-stream") out_sock = ctx.socket(zmq.PUB) -out_sock.connect('ipc:///tmp/zhttp-test-in') +out_sock.connect("ipc:///tmp/zhttp-test-in") while True: - m_raw = in_sock.recv() - req = tnetstring.loads(m_raw[1:]) - print 'IN %s' % req - - out_seq = 0 - - resp = dict() - resp['from'] = client_id - resp['id'] = req['id'] - resp['seq'] = out_seq - out_seq += 1 - resp['type'] = 'handoff-start' - print 'OUT %s' % resp - out_sock.send(req['from'] + ' T' + tnetstring.dumps(resp)) - - m_raw = in_stream_sock.recv_multipart() - req = tnetstring.loads(m_raw[2][1:]) - print 'IN %s' % req - - assert(req['type'] == 'handoff-proceed') - - time.sleep(10) - - resp = dict() - resp['from'] = client_id - resp['id'] = req['id'] - resp['seq'] = out_seq - out_seq += 1 - resp['type'] = 'keep-alive' - print 'OUT %s' % resp - out_sock.send(req['from'] + ' T' + tnetstring.dumps(resp)) - - m_raw = in_stream_sock.recv_multipart() - req = tnetstring.loads(m_raw[2][1:]) - print 'IN %s' % req - - assert(req['type'] == 'error') - print 'error: %s' % req['condition'] + m_raw = in_sock.recv() + req = tnetstring.loads(m_raw[1:]) + print("IN %s" % req) + + out_seq = 0 + + resp = dict() + resp["from"] = client_id + resp["id"] = req["id"] + resp["seq"] = out_seq + out_seq += 1 + resp["type"] = "handoff-start" + print("OUT %s" % resp) + out_sock.send(req["from"] + " T" + tnetstring.dumps(resp)) + + m_raw = in_stream_sock.recv_multipart() + req = tnetstring.loads(m_raw[2][1:]) + print("IN %s" % req) + + assert req["type"] == "handoff-proceed" + + time.sleep(10) + + resp = dict() + resp["from"] = client_id + resp["id"] = req["id"] + resp["seq"] = out_seq + out_seq += 1 + resp["type"] = "keep-alive" + print("OUT %s" % resp) + out_sock.send(req["from"] + " T" + tnetstring.dumps(resp)) + + m_raw = in_stream_sock.recv_multipart() + req = tnetstring.loads(m_raw[2][1:]) + print("IN %s" % req) + + assert req["type"] == "error" + print("error: %s" % req["condition"]) diff --git a/src/m2adapter/tools/hanghandler.py b/src/m2adapter/tools/hanghandler.py index 182ea33d8..aa179d40e 100644 --- a/src/m2adapter/tools/hanghandler.py +++ b/src/m2adapter/tools/hanghandler.py @@ -5,9 +5,9 @@ ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc:///tmp/zhttp-test-out') +in_sock.connect("ipc:///tmp/zhttp-test-out") while True: - m_raw = in_sock.recv() - req = tnetstring.loads(m_raw[1:]) - print 'IN %s' % req + m_raw = in_sock.recv() + req = tnetstring.loads(m_raw[1:]) + print("IN %s" % req) diff --git a/src/m2adapter/tools/keephandler.py b/src/m2adapter/tools/keephandler.py index bfa08a89c..9d6601ce4 100644 --- a/src/m2adapter/tools/keephandler.py +++ b/src/m2adapter/tools/keephandler.py @@ -5,63 +5,68 @@ import tnetstring import zmq + class Session(object): - def __init__(self): - self.to_address = None - self.out_seq = 0 + def __init__(self): + self.to_address = None + self.out_seq = 0 + sessions = dict() lock = threading.Lock() -client_id = 'zhttp-test' +client_id = "zhttp-test" ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc:///tmp/zhttp-test-out') +in_sock.connect("ipc:///tmp/zhttp-test-out") out_sock = ctx.socket(zmq.PUB) -out_sock.connect('ipc:///tmp/zhttp-test-in') +out_sock.connect("ipc:///tmp/zhttp-test-in") + def keepalive_worker(): - lock.acquire() - for id, s in sessions.iteritems(): - resp = dict() - resp['from'] = client_id - resp['id'] = id - resp['seq'] = s.out_seq - s.out_seq += 1 - resp['type'] = 'credit' - resp['credits'] = 0 - - print 'OUT %s' % resp - out_sock.send(s.to_address + ' T' + tnetstring.dumps(resp)) - lock.release() + lock.acquire() + for id, s in sessions.iteritems(): + resp = dict() + resp["from"] = client_id + resp["id"] = id + resp["seq"] = s.out_seq + s.out_seq += 1 + resp["type"] = "credit" + resp["credits"] = 0 + + print("OUT %s" % resp) + out_sock.send(s.to_address + " T" + tnetstring.dumps(resp)) + lock.release() + class KeepAliveThread(threading.Thread): - def run(self): - while True: - keepalive_worker() - time.sleep(30) + def run(self): + while True: + keepalive_worker() + time.sleep(30) + keepalive_thread = KeepAliveThread() keepalive_thread.daemon = True keepalive_thread.start() while True: - m_raw = in_sock.recv() - req = tnetstring.loads(m_raw[1:]) - print 'IN %s' % req - - id = req['id'] - if id in sessions: - print 'session already active' - continue - - if 'type' in req: - print 'wrong packet type' - continue - - s = Session() - s.to_address = req['from'] - lock.acquire() - sessions[id] = s - lock.release() + m_raw = in_sock.recv() + req = tnetstring.loads(m_raw[1:]) + print("IN %s" % req) + + id = req["id"] + if id in sessions: + print("session already active") + continue + + if "type" in req: + print("wrong packet type") + continue + + s = Session() + s.to_address = req["from"] + lock.acquire() + sessions[id] = s + lock.release() diff --git a/src/m2adapter/tools/m2control.py b/src/m2adapter/tools/m2control.py index b6a728b56..8b0055cfc 100644 --- a/src/m2adapter/tools/m2control.py +++ b/src/m2adapter/tools/m2control.py @@ -4,22 +4,22 @@ import zmq import tnetstring from pprint import pprint - + CTX = zmq.Context() addr = sys.argv[1] - + ctl = CTX.socket(zmq.REQ) - -print 'CONNECTING' + +print("CONNECTING") ctl.connect(addr) - + while True: - cmd = raw_input('> ') - # will only work with simple commands that have no arguments - ctl.send(tnetstring.dumps([cmd, {}])) + cmd = raw_input("> ") + # will only work with simple commands that have no arguments + ctl.send(tnetstring.dumps([cmd, {}])) - resp = ctl.recv() + resp = ctl.recv() - pprint(tnetstring.loads(resp)) + pprint(tnetstring.loads(resp)) ctl.close() diff --git a/src/m2adapter/tools/streamhandler.py b/src/m2adapter/tools/streamhandler.py index 776e01321..8a3766519 100644 --- a/src/m2adapter/tools/streamhandler.py +++ b/src/m2adapter/tools/streamhandler.py @@ -5,130 +5,132 @@ import tnetstring import zmq + class Session(object): - def __init__(self): - self.file = None - self.out_seq = 0 - self.credits = 0 + def __init__(self): + self.file = None + self.out_seq = 0 + self.credits = 0 + filename = sys.argv[1] sessions = dict() -client_id = 'zhttp-test' +client_id = "zhttp-test" ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc:///tmp/zhttp-test-out') +in_sock.connect("ipc:///tmp/zhttp-test-out") in_stream_sock = ctx.socket(zmq.DEALER) in_stream_sock.setsockopt(zmq.IDENTITY, client_id) -in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream') +in_stream_sock.connect("ipc:///tmp/zhttp-test-out-stream") out_sock = ctx.socket(zmq.PUB) -out_sock.connect('ipc:///tmp/zhttp-test-in') +out_sock.connect("ipc:///tmp/zhttp-test-in") poller = zmq.Poller() poller.register(in_sock, zmq.POLLIN) poller.register(in_stream_sock, zmq.POLLIN) while True: - socks = dict(poller.poll()) - if socks.get(in_sock) == zmq.POLLIN: - m_raw = in_sock.recv() - req = tnetstring.loads(m_raw[1:]) - print 'IN %s' % req - - id = req['id'] - if id in sessions: - print 'session already active' - continue - - if 'type' in req: - print 'wrong packet type' - continue - - s = Session() - sessions[id] = s - - s.file = open(filename, 'r') - - if 'credits' in req: - s.credits += req['credits'] - - body = '' - eof = False - while s.credits > 0: - buf = s.file.read(s.credits) - s.credits -= len(buf) - body += buf - if len(buf) == 0: - eof = True - break - - resp = dict() - resp['from'] = client_id - resp['id'] = req['id'] - resp['seq'] = s.out_seq - resp['code'] = 200 - resp['reason'] = 'OK' - headers = list() - headers.append(['Content-Type', 'text/plain']) - headers.append(['Content-Length', str(os.stat(filename).st_size)]) - resp['headers'] = headers - if len(body) > 0: - resp['body'] = body - if not eof: - resp['more'] = True - - s.out_seq += 1 - - if eof: - del sessions[id] - - print 'OUT %s' % resp - out_sock.send(req['from'] + ' T' + tnetstring.dumps(resp)) - elif socks.get(in_stream_sock) == zmq.POLLIN: - parts = in_stream_sock.recv_multipart() - req = tnetstring.loads(parts[1][1:]) - print 'IN stream %s' % req - - # we are only streaming output, so subsequent input messages must be credits - if 'type' not in req or req['type'] != 'credit': - print 'wrong packet type' - continue - - id = req['id'] - - s = sessions.get(id) - if s is None: - continue - - if 'credits' in req: - s.credits += req['credits'] - - body = '' - eof = False - while s.credits > 0: - buf = s.file.read(s.credits) - s.credits -= len(buf) - body += buf - if len(buf) == 0: - eof = True - break - - resp = dict() - resp['from'] = client_id - resp['id'] = id - resp['seq'] = s.out_seq - - s.out_seq += 1 - - if len(body) > 0: - resp['body'] = body - if not eof: - resp['more'] = True - - if eof: - del sessions[id] - - print 'OUT %s' % resp - out_sock.send(req['from'] + ' T' + tnetstring.dumps(resp)) + socks = dict(poller.poll()) + if socks.get(in_sock) == zmq.POLLIN: + m_raw = in_sock.recv() + req = tnetstring.loads(m_raw[1:]) + print("IN %s" % req) + + id = req["id"] + if id in sessions: + print("session already active") + continue + + if "type" in req: + print("wrong packet type") + continue + + s = Session() + sessions[id] = s + + s.file = open(filename, "r") + + if "credits" in req: + s.credits += req["credits"] + + body = "" + eof = False + while s.credits > 0: + buf = s.file.read(s.credits) + s.credits -= len(buf) + body += buf + if len(buf) == 0: + eof = True + break + + resp = dict() + resp["from"] = client_id + resp["id"] = req["id"] + resp["seq"] = s.out_seq + resp["code"] = 200 + resp["reason"] = "OK" + headers = list() + headers.append(["Content-Type", "text/plain"]) + headers.append(["Content-Length", str(os.stat(filename).st_size)]) + resp["headers"] = headers + if len(body) > 0: + resp["body"] = body + if not eof: + resp["more"] = True + + s.out_seq += 1 + + if eof: + del sessions[id] + + print("OUT %s" % resp) + out_sock.send(req["from"] + " T" + tnetstring.dumps(resp)) + elif socks.get(in_stream_sock) == zmq.POLLIN: + parts = in_stream_sock.recv_multipart() + req = tnetstring.loads(parts[1][1:]) + print("IN stream %s" % req) + + # we are only streaming output, so subsequent input messages must be credits + if "type" not in req or req["type"] != "credit": + print("wrong packet type") + continue + + id = req["id"] + + s = sessions.get(id) + if s is None: + continue + + if "credits" in req: + s.credits += req["credits"] + + body = "" + eof = False + while s.credits > 0: + buf = s.file.read(s.credits) + s.credits -= len(buf) + body += buf + if len(buf) == 0: + eof = True + break + + resp = dict() + resp["from"] = client_id + resp["id"] = id + resp["seq"] = s.out_seq + + s.out_seq += 1 + + if len(body) > 0: + resp["body"] = body + if not eof: + resp["more"] = True + + if eof: + del sessions[id] + + print("OUT %s" % resp) + out_sock.send(req["from"] + " T" + tnetstring.dumps(resp)) diff --git a/src/m2adapter/tools/streamposthandler.py b/src/m2adapter/tools/streamposthandler.py index c0aedea7a..c044a786b 100644 --- a/src/m2adapter/tools/streamposthandler.py +++ b/src/m2adapter/tools/streamposthandler.py @@ -3,142 +3,144 @@ import tnetstring import zmq + class Session(object): - def __init__(self): - self.id = None - # 0 = receiving, 1 = responding - self.state = 0 - self.sent_credits = False - self.content_type = None - self.data = '' - self.offset = 0 - self.out_seq = 0 - self.credits = 0 - - def take_data(self): - left = len(self.data) - self.offset - if self.credits >= left: - take = left - else: - take = self.credits - body = self.data[self.offset:self.offset + take] - self.offset += take - self.credits -= take - return body + def __init__(self): + self.id = None + # 0 = receiving, 1 = responding + self.state = 0 + self.sent_credits = False + self.content_type = None + self.data = "" + self.offset = 0 + self.out_seq = 0 + self.credits = 0 + + def take_data(self): + left = len(self.data) - self.offset + if self.credits >= left: + take = left + else: + take = self.credits + body = self.data[self.offset : self.offset + take] + self.offset += take + self.credits -= take + return body + sessions = dict() -client_id = 'zhttp-test' +client_id = "zhttp-test" ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc:///tmp/zhttp-test-out') +in_sock.connect("ipc:///tmp/zhttp-test-out") in_stream_sock = ctx.socket(zmq.DEALER) in_stream_sock.setsockopt(zmq.IDENTITY, client_id) -in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream') +in_stream_sock.connect("ipc:///tmp/zhttp-test-out-stream") out_sock = ctx.socket(zmq.PUB) -out_sock.connect('ipc:///tmp/zhttp-test-in') +out_sock.connect("ipc:///tmp/zhttp-test-in") poller = zmq.Poller() poller.register(in_sock, zmq.POLLIN) poller.register(in_stream_sock, zmq.POLLIN) while True: - socks = dict(poller.poll()) - if socks.get(in_sock) == zmq.POLLIN: - m_raw = in_sock.recv() - req = tnetstring.loads(m_raw[1:]) - print 'IN %s' % req - - id = req['id'] - if id in sessions: - print 'session already active' - continue - - s = Session() - s.id = id - headers = req.get('headers') - if headers: - for h in headers: - if h[0].lower() == 'content-type': - s.content_type = h[1] - break - sessions[id] = s - - elif socks.get(in_stream_sock) == zmq.POLLIN: - parts = in_stream_sock.recv_multipart() - req = tnetstring.loads(parts[1][1:]) - print 'IN stream %s' % req - - s = sessions.get(id) - if s is None: - print 'no such session' - continue - - ptype = req.get('type') - if ptype is None or (ptype is not None and ptype == 'credit'): - if 'credits' in req: - s.credits += req['credits'] - - if ptype is None: - assert(s.state == 0) - body = '' - if 'body' in req: - body = req['body'] - - s.data += body - - if not req.get('more'): - s.state = 1 # responding - - resp = dict() - resp['from'] = client_id - resp['id'] = s.id - resp['seq'] = s.out_seq - s.out_seq += 1 - resp['code'] = 200 - resp['reason'] = 'OK' - headers = list() - if s.content_type: - headers.append(['Content-Type', s.content_type]) - headers.append(['Content-Length', str(len(s.data))]) - resp['headers'] = headers - resp['body'] = s.take_data() - if s.offset < len(s.data): - resp['more'] = True - else: - del sessions[s.id] - - print 'OUT %s' % resp - out_sock.send(req['from'] + ' T' + tnetstring.dumps(resp)) - else: - # send credits - resp = dict() - resp['from'] = client_id - resp['id'] = s.id - resp['seq'] = s.out_seq - s.out_seq += 1 - resp['type'] = 'credit' - if not s.sent_credits: - resp['credits'] = 200000 - s.sent_credits = True - else: - resp['credits'] = len(body) - - print 'OUT %s' % resp - out_sock.send(req['from'] + ' T' + tnetstring.dumps(resp)) - elif ptype is not None and ptype == 'credit': - if s.state == 1 and s.credits > 0: - resp = dict() - resp['from'] = client_id - resp['id'] = id - resp['seq'] = s.out_seq - s.out_seq += 1 - resp['body'] = s.take_data() - if s.offset < len(s.data): - resp['more'] = True - else: - del sessions[s.id] - - print 'OUT %s' % resp - out_sock.send(req['from'] + ' T' + tnetstring.dumps(resp)) + socks = dict(poller.poll()) + if socks.get(in_sock) == zmq.POLLIN: + m_raw = in_sock.recv() + req = tnetstring.loads(m_raw[1:]) + print("IN %s" % req) + + id = req["id"] + if id in sessions: + print("session already active") + continue + + s = Session() + s.id = id + headers = req.get("headers") + if headers: + for h in headers: + if h[0].lower() == "content-type": + s.content_type = h[1] + break + sessions[id] = s + + elif socks.get(in_stream_sock) == zmq.POLLIN: + parts = in_stream_sock.recv_multipart() + req = tnetstring.loads(parts[1][1:]) + print("IN stream %s" % req) + + s = sessions.get(id) + if s is None: + print("no such session") + continue + + ptype = req.get("type") + if ptype is None or (ptype is not None and ptype == "credit"): + if "credits" in req: + s.credits += req["credits"] + + if ptype is None: + assert s.state == 0 + body = "" + if "body" in req: + body = req["body"] + + s.data += body + + if not req.get("more"): + s.state = 1 # responding + + resp = dict() + resp["from"] = client_id + resp["id"] = s.id + resp["seq"] = s.out_seq + s.out_seq += 1 + resp["code"] = 200 + resp["reason"] = "OK" + headers = list() + if s.content_type: + headers.append(["Content-Type", s.content_type]) + headers.append(["Content-Length", str(len(s.data))]) + resp["headers"] = headers + resp["body"] = s.take_data() + if s.offset < len(s.data): + resp["more"] = True + else: + del sessions[s.id] + + print("OUT %s" % resp) + out_sock.send(req["from"] + " T" + tnetstring.dumps(resp)) + else: + # send credits + resp = dict() + resp["from"] = client_id + resp["id"] = s.id + resp["seq"] = s.out_seq + s.out_seq += 1 + resp["type"] = "credit" + if not s.sent_credits: + resp["credits"] = 200000 + s.sent_credits = True + else: + resp["credits"] = len(body) + + print("OUT %s" % resp) + out_sock.send(req["from"] + " T" + tnetstring.dumps(resp)) + elif ptype is not None and ptype == "credit": + if s.state == 1 and s.credits > 0: + resp = dict() + resp["from"] = client_id + resp["id"] = id + resp["seq"] = s.out_seq + s.out_seq += 1 + resp["body"] = s.take_data() + if s.offset < len(s.data): + resp["more"] = True + else: + del sessions[s.id] + + print("OUT %s" % resp) + out_sock.send(req["from"] + " T" + tnetstring.dumps(resp)) diff --git a/src/m2adapter/tools/testhandler.py b/src/m2adapter/tools/testhandler.py index 2a2e503f4..971c05492 100644 --- a/src/m2adapter/tools/testhandler.py +++ b/src/m2adapter/tools/testhandler.py @@ -5,21 +5,21 @@ ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc:///tmp/zhttp-test-out') +in_sock.connect("ipc:///tmp/zhttp-test-out") out_sock = ctx.socket(zmq.PUB) -out_sock.connect('ipc:///tmp/zhttp-test-in') +out_sock.connect("ipc:///tmp/zhttp-test-in") while True: - m_raw = in_sock.recv() - req = tnetstring.loads(m_raw[1:]) - print 'IN %s' % req + m_raw = in_sock.recv() + req = tnetstring.loads(m_raw[1:]) + print("IN %s" % req) - resp = dict() - resp['id'] = req['id'] - resp['code'] = 200 - resp['reason'] = 'OK' - resp['headers'] = [['Content-Type', 'text/plain']] - resp['body'] = 'hello world\n' + resp = dict() + resp["id"] = req["id"] + resp["code"] = 200 + resp["reason"] = "OK" + resp["headers"] = [["Content-Type", "text/plain"]] + resp["body"] = "hello world\n" - print 'OUT %s' % resp - out_sock.send(req['from'] + ' T' + tnetstring.dumps(resp)) + print("OUT %s" % resp) + out_sock.send(req["from"] + " T" + tnetstring.dumps(resp)) diff --git a/src/m2adapter/tools/wsechohandler.py b/src/m2adapter/tools/wsechohandler.py index 2fd52ff5b..4970cda06 100644 --- a/src/m2adapter/tools/wsechohandler.py +++ b/src/m2adapter/tools/wsechohandler.py @@ -3,16 +3,16 @@ import tnetstring import zmq -instance_id = 'wsechohandler' +instance_id = "wsechohandler" ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc:///tmp/zhttp-test-out') +in_sock.connect("ipc:///tmp/zhttp-test-out") in_stream_sock = ctx.socket(zmq.ROUTER) in_stream_sock.setsockopt(zmq.IDENTITY, instance_id) -in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream') +in_stream_sock.connect("ipc:///tmp/zhttp-test-out-stream") out_sock = ctx.socket(zmq.PUB) -out_sock.connect('ipc:///tmp/zhttp-test-in') +out_sock.connect("ipc:///tmp/zhttp-test-in") poller = zmq.Poller() poller.register(in_sock, zmq.POLLIN) @@ -21,51 +21,51 @@ sessions = set() while True: - socks = dict(poller.poll()) - if socks.get(in_sock) == zmq.POLLIN: - m_raw = in_sock.recv() - req = tnetstring.loads(m_raw[1:]) - elif socks.get(in_stream_sock) == zmq.POLLIN: - m_raw = in_stream_sock.recv_multipart() - req = tnetstring.loads(m_raw[2][1:]) - else: - continue + socks = dict(poller.poll()) + if socks.get(in_sock) == zmq.POLLIN: + m_raw = in_sock.recv() + req = tnetstring.loads(m_raw[1:]) + elif socks.get(in_stream_sock) == zmq.POLLIN: + m_raw = in_stream_sock.recv_multipart() + req = tnetstring.loads(m_raw[2][1:]) + else: + continue - print 'IN %s' % req + print("IN %s" % req) - rid = (req['from'], req['id']) + rid = (req["from"], req["id"]) - resp = dict() - if rid not in sessions: - rtype = req.get('type') + resp = dict() + if rid not in sessions: + rtype = req.get("type") - # first packet must be a data packet - if rtype is not None: - continue + # first packet must be a data packet + if rtype is not None: + continue - sessions.add(rid) - resp['credits'] = 200000 - else: - rtype = req.get('type') - if rtype is None: - if 'content-type' in req: - resp['content-type'] = req['content-type'] - resp['body'] = req['body'] - count = len(req['body']) - if req.get('more'): - resp['more'] = True - resp['credits'] = len(req['body']) - elif rtype == 'close': - sessions.remove(rid) - resp['type'] = 'close' - if 'code' in req: - resp['code'] = req['code'] - elif rtype == 'keep-alive': - resp['type'] = 'keep-alive' - else: - continue + sessions.add(rid) + resp["credits"] = 200000 + else: + rtype = req.get("type") + if rtype is None: + if "content-type" in req: + resp["content-type"] = req["content-type"] + resp["body"] = req["body"] + count = len(req["body"]) + if req.get("more"): + resp["more"] = True + resp["credits"] = len(req["body"]) + elif rtype == "close": + sessions.remove(rid) + resp["type"] = "close" + if "code" in req: + resp["code"] = req["code"] + elif rtype == "keep-alive": + resp["type"] = "keep-alive" + else: + continue - resp['from'] = instance_id - resp['id'] = req['id'] - print 'OUT %s' % resp - out_sock.send(req['from'] + ' T' + tnetstring.dumps(resp)) + resp["from"] = instance_id + resp["id"] = req["id"] + print("OUT %s" % resp) + out_sock.send(req["from"] + " T" + tnetstring.dumps(resp)) diff --git a/src/proxy/app.cpp b/src/proxy/app.cpp index 98cfa64b8..b2bf71888 100644 --- a/src/proxy/app.cpp +++ b/src/proxy/app.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2012-2022 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -30,13 +30,17 @@ #include #include #include "processquit.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "log.h" #include "settings.h" #include "xffrule.h" #include "domainmap.h" #include "engine.h" #include "config.h" +#include "cacheutil.h" + +extern bool gCacheThreadAllowFlag; using Connection = boost::signals2::scoped_connection; @@ -298,7 +302,8 @@ class EngineThread : public QThread QCoreApplication::instance()->sendPostedEvents(); // deinit here, after all event loop activity has completed - RTimer::deinit(); + Timer::deinit(); + DeferCall::cleanup(); } private: @@ -424,7 +429,11 @@ class App::Private : public QObject QStringList services = settings.value("runner/services").toStringList(); + bool cacheEnable = settings.value("cache/cache_enable").toBool(); + int workerCount = settings.value("proxy/workers", 1).toInt(); + if (cacheEnable == true) + workerCount = 1; QStringList connmgr_in_specs = settings.value("proxy/connmgr_in_specs").toStringList(); trimlist(&connmgr_in_specs); QStringList connmgr_in_stream_specs = settings.value("proxy/connmgr_in_stream_specs").toStringList(); @@ -622,6 +631,137 @@ class App::Private : public QObject config.prometheusPort = prometheusPort; config.prometheusPrefix = prometheusPrefix; + // Cache config + QStringList httpBackendUrlList = settings.value("cache/http_backend_urls").toStringList(); + QStringList wsBackendUrlList = settings.value("cache/ws_backend_urls").toStringList(); + QStringList cacheMethodList = settings.value("cache/ws_cache_methods").toStringList(); + QStringList subscribeMethodList = settings.value("cache/ws_subscribe_methods").toStringList(); + QStringList neverTimeoutMethodList = settings.value("cache/ws_never_timeout_methods").toStringList(); + QStringList refreshShorterMethodList = settings.value("cache/ws_refresh_shorter_methods").toStringList(); + QStringList refreshLongerMethodList = settings.value("cache/ws_refresh_longer_methods").toStringList(); + QStringList refreshUneraseMethodList = settings.value("cache/ws_refresh_unerase_methods").toStringList(); + QStringList refreshExcludeMethodList = settings.value("cache/ws_refresh_exclude_methods").toStringList(); + QStringList refreshPassthroughMethodList = settings.value("cache/ws_refresh_passthrough_methods").toStringList(); + QStringList nullResponseMethodList = settings.value("cache/ws_null_response_methods").toStringList(); + QString cacheKeyConfig = settings.value("cache/ws_cache_key", "").toString().simplified().remove("'").remove("\"").toLower(); + QStringList cacheKeyParts = cacheKeyConfig.split(u'+', QString::SkipEmptyParts); + QStringList cacheKeyItemList; + for (int i = 0; i < cacheKeyParts.count(); i++) + { + QString keyPart = cacheKeyParts[i].trimmed(); + if (keyPart.startsWith("$request_json_value[") && keyPart.endsWith("]")) + { + QString jsonValue = keyPart.mid(20, keyPart.length()-20-1).trimmed(); + jsonValue += ".JSON_VALUE"; + cacheKeyItemList.append(jsonValue); + } + else if (keyPart.startsWith("$request_json_pair[") && keyPart.endsWith("]")) + { + QString jsonValue = keyPart.mid(19, keyPart.length()-19-1).trimmed(); + jsonValue += ".JSON_PAIR"; + cacheKeyItemList.append(jsonValue); + } + else if (keyPart.startsWith("$user_defined[") && keyPart.endsWith("]")) + { + QString jsonValue = keyPart.mid(14, keyPart.length()-14-1).trimmed(); + QString userDefinedKeyConfig = settings.value("cache/"+jsonValue, "").toString().simplified().remove("'").remove("\"").toLower(); + QStringList userDefinedKeyParts = userDefinedKeyConfig.split(u'+', QString::SkipEmptyParts); + for (int j = 0; j < userDefinedKeyParts.count(); j++) + { + QString userDefinedKeyPart = userDefinedKeyParts[j].trimmed(); + if (userDefinedKeyPart.startsWith("$request_json_value[") && userDefinedKeyPart.endsWith("]")) + { + jsonValue = userDefinedKeyPart.mid(20, userDefinedKeyPart.length()-20-1).trimmed(); + jsonValue += ".JSON_VALUE"; + cacheKeyItemList.append(jsonValue); + } + else if (userDefinedKeyPart.startsWith("$request_json_pair[") && userDefinedKeyPart.endsWith("]")) + { + jsonValue = userDefinedKeyPart.mid(19, userDefinedKeyPart.length()-19-1).trimmed(); + jsonValue += ".JSON_PAIR"; + cacheKeyItemList.append(jsonValue); + } + else + { + userDefinedKeyPart += ".RAW_VALUE"; + cacheKeyItemList.append(userDefinedKeyPart); + } + } + } + else + { + keyPart += ".RAW_VALUE"; + cacheKeyItemList.append(keyPart); + } + } + // message iden attribute and cache check attribute + QString msgIdFieldName = settings.value("cache/message_id_attribute", "").toString().simplified().remove("'").remove("\"").toLower(); + QString msgMethodFieldName = settings.value("cache/message_method_attribute", "").toString().simplified().remove("'").remove("\"").toLower(); + QString msgParamsFieldName = settings.value("cache/message_params_attribute", "params").toString().simplified().remove("'").remove("\"").toLower(); + QStringList msgErrorFieldList = settings.value("cache/message_error_attributes").toStringList(); + // cache shorter timeout seconds (default 6) + int cacheTimeoutSeconds = settings.value("cache/ws_auto_refresh_cache_timeout_seconds", 20).toInt(); + int shorterTimeoutSeconds = settings.value("cache/ws_auto_refresh_shorter_cache_timeout_seconds", 10).toInt(); + int longerTimeoutSeconds = settings.value("cache/ws_auto_refresh_longer_timeout_seconds", 60).toInt(); + int accessTimeoutSeconds = settings.value("cache/ws_auto_refresh_access_timeout_seconds", 30).toInt(); + // Maximum cache item count (default 3000) + int cacheItemMaxCount = settings.value("cache/cache_item_max_count", 3000).toInt(); + // time seconds to retry another backend for null response (default 10) + int backendSwitchIntervalSeconds = settings.value("cache/backend_switch_interval_seconds", 10).toInt(); + // prometheus restore allow seconds (default 300) + int prometheusRestoreAllowSeconds = settings.value("cache/prometheus_restore_allow_seconds", 300).toInt(); + // redis + bool redisEnable = settings.value("cache/redis_enable").toBool(); + QString redisHostAddr = settings.value("cache/redis_host_addr").toString(); + int redisPort = settings.value("cache/redis_port", 6379).toInt(); + int redisPoolCount = settings.value("cache/redis_pool_count", 10).toInt(); + QString redisKeyHeader = settings.value("cache/redis_key_header").toString(); + QString replicaMasterAddr = settings.value("cache/replica_master_addr").toString(); + int replicaMasterPort = settings.value("cache/replica_master_port", 6379).toInt(); + // count method group + QStringList countMethodGroups = settings.value("cache/ws_count_groups").toStringList(); + QMap countMethodGroupMap; + for (int i = 0; i < countMethodGroups.count(); i++) + { + QString groupKey = countMethodGroups[i]; + QStringList groupValue = settings.value("cache/" + groupKey).toStringList(); + countMethodGroupMap[groupKey] = groupValue; + } + + config.cacheEnable = cacheEnable; + config.httpBackendUrlList = httpBackendUrlList; + config.wsBackendUrlList = wsBackendUrlList; + config.cacheMethodList = cacheMethodList; + config.subscribeMethodList = subscribeMethodList; + config.neverTimeoutMethodList = neverTimeoutMethodList; + config.refreshShorterMethodList = refreshShorterMethodList; + config.refreshLongerMethodList = refreshLongerMethodList; + config.refreshUneraseMethodList = refreshUneraseMethodList; + config.refreshExcludeMethodList = refreshExcludeMethodList; + config.refreshPassthroughMethodList = refreshPassthroughMethodList; + config.nullResponseMethodList = nullResponseMethodList; + config.cacheKeyItemList = cacheKeyItemList; + config.msgIdFieldName = msgIdFieldName; + config.msgMethodFieldName = msgMethodFieldName; + config.msgParamsFieldName = msgParamsFieldName; + config.msgErrorFieldList = msgErrorFieldList; + config.cacheTimeoutSeconds = cacheTimeoutSeconds; + config.cacheItemMaxCount = cacheItemMaxCount; + config.shorterTimeoutSeconds = shorterTimeoutSeconds; + config.longerTimeoutSeconds = longerTimeoutSeconds; + config.accessTimeoutSeconds = accessTimeoutSeconds; + config.backendSwitchIntervalSeconds = backendSwitchIntervalSeconds; + config.prometheusRestoreAllowSeconds = prometheusRestoreAllowSeconds; + config.redisEnable = redisEnable; + config.redisEnable = redisEnable; + config.redisHostAddr = redisHostAddr; + config.redisPort = redisPort; + config.redisPoolCount = redisPoolCount; + config.redisKeyHeader = redisKeyHeader; + config.replicaMasterAddr = replicaMasterAddr; + config.replicaMasterPort = replicaMasterPort; + config.countMethodGroupMap = countMethodGroupMap; + for(int n = 0; n < workerCount; ++n) { Engine::Configuration wconfig = config; @@ -664,6 +804,13 @@ class App::Private : public QObject log_info("started"); } +private: + void domainMap_changed() + { + for(EngineThread *t : threads) + t->routesChanged(); + } + private slots: void reload() { @@ -673,16 +820,13 @@ private slots: domainMap->reload(); } - void domainMap_changed() - { - for(EngineThread *t : threads) - t->routesChanged(); - } - void doQuit() { log_info("stopping..."); + gCacheThreadAllowFlag = false; + save_prometheusStatIntoFile(); + // remove the handler, so if we get another signal then we crash out ProcessQuit::cleanup(); diff --git a/src/proxy/domainmap.cpp b/src/proxy/domainmap.cpp index 2446b8e2c..898d7362b 100644 --- a/src/proxy/domainmap.cpp +++ b/src/proxy/domainmap.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2012-2022 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -34,7 +34,8 @@ #include #include #include "log.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "routesfile.h" #define WORKER_THREAD_TIMERS 1 @@ -201,9 +202,10 @@ class DomainMap::Worker : public QObject QList allRules; QHash< QString, QList > rulesByDomain; QHash rulesById; - RTimer t; + Timer t; Connection tConnection; QFileSystemWatcher watcher; + DeferCall deferCall; Worker() : watcher(this) @@ -291,7 +293,7 @@ class DomainMap::Worker : public QObject log_info("routes loaded with %d entries", allRules.count()); - QMetaObject::invokeMethod(this, "doChanged", Qt::QueuedConnection); + deferCall.defer([=] { doChanged(); }); } // mutex must be locked when calling this method @@ -311,11 +313,6 @@ class DomainMap::Worker : public QObject Signal changed; public slots: - void doChanged() - { - changed(); - } - void start() { if(!fileName.isEmpty()) @@ -718,6 +715,11 @@ public slots: return AddRuleOk; } + + void doChanged() + { + changed(); + } }; class DomainMap::Thread : public QThread @@ -738,6 +740,8 @@ class DomainMap::Thread : public QThread void start() { + setObjectName("domainmap"); + QMutexLocker locker(&m); QThread::start(); w.wait(&m); @@ -745,7 +749,7 @@ class DomainMap::Thread : public QThread virtual void run() { - RTimer::init(WORKER_THREAD_TIMERS); + Timer::init(WORKER_THREAD_TIMERS); worker = new Worker; worker->fileName = fileName; @@ -755,7 +759,8 @@ class DomainMap::Thread : public QThread startedConnection.disconnect(); delete worker; - RTimer::deinit(); + Timer::deinit(); + DeferCall::cleanup(); } public: diff --git a/src/proxy/engine.cpp b/src/proxy/engine.cpp index 6f578a8ac..61e6241a6 100644 --- a/src/proxy/engine.cpp +++ b/src/proxy/engine.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2012-2023 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -33,7 +33,8 @@ #include "packet/statspacket.h" #include "packet/zrpcrequestpacket.h" #include "qtcompat.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "log.h" #include "inspectdata.h" #include "zhttpmanager.h" @@ -127,8 +128,8 @@ class Engine::Private : public QObject StatsManager *stats; ZrpcManager *command; ZrpcManager *accept; - QZmq::Socket *handler_retry_in_sock; - QZmq::Valve *handler_retry_in_valve; + std::unique_ptr handler_retry_in_sock; + std::unique_ptr handler_retry_in_valve; QSet requestSessions; QHash proxyItemsByKey; QHash proxyItemsBySession; @@ -160,8 +161,6 @@ class Engine::Private : public QObject stats(0), command(0), accept(0), - handler_retry_in_sock(0), - handler_retry_in_valve(0), sockJsManager(0), updater(0) { @@ -219,7 +218,7 @@ class Engine::Private : public QObject config = _config; // enough timers for sessions and zroutes, plus an extra 100 for misc - RTimer::init((config.sessionsMax * TIMERS_PER_SESSION) + (ZROUTES_MAX * TIMERS_PER_ZROUTE) + 100); + Timer::init((config.sessionsMax * TIMERS_PER_SESSION) + (ZROUTES_MAX * TIMERS_PER_ZROUTE) + 100); logConfig.fromAddress = config.logFrom; logConfig.userAgent = config.logUserAgent; @@ -234,6 +233,39 @@ class Engine::Private : public QObject zhttpIn->setServerInSpecs(config.serverInSpecs); zhttpIn->setServerInStreamSpecs(config.serverInStreamSpecs); zhttpIn->setServerOutSpecs(config.serverOutSpecs); + zhttpIn->setCacheParameters( + config.cacheEnable, + config.httpBackendUrlList, + config.wsBackendUrlList, + config.cacheMethodList, + config.subscribeMethodList, + config.neverTimeoutMethodList, + config.refreshShorterMethodList, + config.refreshLongerMethodList, + config.refreshUneraseMethodList, + config.refreshExcludeMethodList, + config.refreshPassthroughMethodList, + config.nullResponseMethodList, + config.cacheKeyItemList, + config.msgIdFieldName, + config.msgMethodFieldName, + config.msgParamsFieldName, + config.msgErrorFieldList, + config.cacheTimeoutSeconds, + config.shorterTimeoutSeconds, + config.longerTimeoutSeconds, + config.accessTimeoutSeconds, + config.cacheItemMaxCount, + config.backendSwitchIntervalSeconds, + config.prometheusRestoreAllowSeconds, + config.redisEnable, + config.redisHostAddr, + config.redisPort, + config.redisPoolCount, + config.redisKeyHeader, + config.replicaMasterAddr, + config.replicaMasterPort, + config.countMethodGroupMap); if(!config.intServerInSpecs.isEmpty() && !config.intServerInStreamSpecs.isEmpty() && !config.intServerOutSpecs.isEmpty()) { @@ -291,19 +323,19 @@ class Engine::Private : public QObject if(!config.retryInSpec.isEmpty()) { - handler_retry_in_sock = new QZmq::Socket(QZmq::Socket::Router, this); + handler_retry_in_sock = std::make_unique(QZmq::Socket::Router); handler_retry_in_sock->setIdentity(config.clientId); handler_retry_in_sock->setHwm(DEFAULT_HWM); QString errorMessage; - if(!ZUtil::setupSocket(handler_retry_in_sock, config.retryInSpec, true, config.ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(handler_retry_in_sock.get(), config.retryInSpec, true, config.ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } - handler_retry_in_valve = new QZmq::Valve(handler_retry_in_sock, this); + handler_retry_in_valve = std::make_unique(handler_retry_in_sock.get()); rrConnection = handler_retry_in_valve->readyRead.connect(boost::bind(&Private::handler_retry_in_readyRead, this, boost::placeholders::_1)); } @@ -360,7 +392,7 @@ class Engine::Private : public QObject if(!stats->setPrometheusPort(config.prometheusPort)) { log_error("unable to bind to prometheus port: %s", qPrintable(config.prometheusPort)); - return false; + //return false; } } } @@ -861,7 +893,7 @@ class Engine::Private : public QObject delete i; ps->finishedByPassthroughCallback().remove(this); - ps->deleteLater(); + DeferCall::deleteLater(ps); tryTakeNext(); } @@ -1026,7 +1058,7 @@ class Engine::Private : public QObject return; } - WebSocketOverHttp *woh = qobject_cast(ps->outSocket()); + WebSocketOverHttp *woh = dynamic_cast(ps->outSocket()); if(woh) woh->refresh(); diff --git a/src/proxy/engine.h b/src/proxy/engine.h index 0fa82b968..83b73960d 100644 --- a/src/proxy/engine.h +++ b/src/proxy/engine.h @@ -95,6 +95,39 @@ class Engine : public QObject QString prometheusPort; QString prometheusPrefix; + bool cacheEnable; + QStringList httpBackendUrlList; + QStringList wsBackendUrlList; + QStringList cacheMethodList; + QStringList subscribeMethodList; + QStringList neverTimeoutMethodList; + QStringList refreshShorterMethodList; + QStringList refreshLongerMethodList; + QStringList refreshUneraseMethodList; + QStringList refreshExcludeMethodList; + QStringList refreshPassthroughMethodList; + QStringList nullResponseMethodList; + QStringList cacheKeyItemList; + QString msgIdFieldName; + QString msgMethodFieldName; + QString msgParamsFieldName; + QStringList msgErrorFieldList; + int cacheTimeoutSeconds; + int shorterTimeoutSeconds; + int longerTimeoutSeconds; + int accessTimeoutSeconds; + int cacheItemMaxCount; + int backendSwitchIntervalSeconds; + int prometheusRestoreAllowSeconds; + bool redisEnable; + QString redisHostAddr; + int redisPort; + int redisPoolCount; + QString redisKeyHeader; + QString replicaMasterAddr; + int replicaMasterPort; + QMap countMethodGroupMap; + Configuration() : id(0), ipcFileMode(-1), @@ -114,7 +147,22 @@ class Engine : public QObject statsConnectionSend(false), statsConnectionTtl(-1), statsConnectionsMaxTtl(-1), - statsReportInterval(-1) + statsReportInterval(-1), + cacheEnable(false), + cacheTimeoutSeconds(20), + shorterTimeoutSeconds(10), + longerTimeoutSeconds(60), + accessTimeoutSeconds(30), + cacheItemMaxCount(3000), + backendSwitchIntervalSeconds(10), + prometheusRestoreAllowSeconds(300), + redisEnable(false), + redisHostAddr("127.0.0.1"), + redisPort(6379), + redisPoolCount(10), + redisKeyHeader(""), + replicaMasterAddr(""), + replicaMasterPort(6379) { } }; diff --git a/src/proxy/main.cpp b/src/proxy/main.cpp index 37f125d47..b50605b92 100644 --- a/src/proxy/main.cpp +++ b/src/proxy/main.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2016 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -22,15 +23,13 @@ #include #include "app.h" +#include "defercall.h" -class AppMain : public QObject +class AppMain { - Q_OBJECT - public: App *app; -public slots: void start() { app = new App; @@ -53,10 +52,17 @@ int proxy_main(int argc, char **argv) QCoreApplication qapp(argc, argv); AppMain appMain; - QMetaObject::invokeMethod(&appMain, "start", Qt::QueuedConnection); - return qapp.exec(); -} + DeferCall deferCall; + deferCall.defer([&] { appMain.start(); }); + int ret = qapp.exec(); + + // ensure deferred deletes are processed + QCoreApplication::instance()->sendPostedEvents(); + // deinit here, after all event loop activity has completed + DeferCall::cleanup(); + + return ret; } -#include "main.moc" +} diff --git a/src/proxy/proxyenginetest.cpp b/src/proxy/proxyenginetest.cpp index e49bfdea6..eeef9cc86 100644 --- a/src/proxy/proxyenginetest.cpp +++ b/src/proxy/proxyenginetest.cpp @@ -37,7 +37,8 @@ #include "packet/httprequestdata.h" #include "packet/httpresponsedata.h" #include "packet/statspacket.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "zhttpmanager.h" #include "statsmanager.h" #include "domainmap.h" @@ -55,21 +56,21 @@ class Wrapper : public QObject Q_OBJECT public: - QZmq::Socket *zhttpClientOutSock; - QZmq::Socket *zhttpClientOutStreamSock; - QZmq::Socket *zhttpClientInSock; - QZmq::Valve *zhttpClientInValve; - QZmq::Socket *zhttpServerInSock; - QZmq::Valve *zhttpServerInValve; - QZmq::Socket *zhttpServerInStreamSock; - QZmq::Valve *zhttpServerInStreamValve; - QZmq::Socket *zhttpServerOutSock; - - QZmq::Socket *handlerInspectSock; - QZmq::Valve *handlerInspectValve; - QZmq::Socket *handlerAcceptSock; - QZmq::Valve *handlerAcceptValve; - QZmq::Socket *handlerRetryOutSock; + std::unique_ptr zhttpClientOutSock; + std::unique_ptr zhttpClientOutStreamSock; + std::unique_ptr zhttpClientInSock; + std::unique_ptr zhttpClientInValve; + std::unique_ptr zhttpServerInSock; + std::unique_ptr zhttpServerInValve; + std::unique_ptr zhttpServerInStreamSock; + std::unique_ptr zhttpServerInStreamValve; + std::unique_ptr zhttpServerOutSock; + + std::unique_ptr handlerInspectSock; + std::unique_ptr handlerInspectValve; + std::unique_ptr handlerAcceptSock; + std::unique_ptr handlerAcceptValve; + std::unique_ptr handlerRetryOutSock; QDir workDir; QHash serverReqs; @@ -107,37 +108,37 @@ class Wrapper : public QObject { // http sockets - zhttpClientOutSock = new QZmq::Socket(QZmq::Socket::Push, this); + zhttpClientOutSock = std::make_unique(QZmq::Socket::Push); - zhttpClientOutStreamSock = new QZmq::Socket(QZmq::Socket::Router, this); + zhttpClientOutStreamSock = std::make_unique(QZmq::Socket::Router); - zhttpClientInSock = new QZmq::Socket(QZmq::Socket::Sub, this); - zhttpClientInValve = new QZmq::Valve(zhttpClientInSock, this); + zhttpClientInSock = std::make_unique(QZmq::Socket::Sub); + zhttpClientInValve = std::make_unique(zhttpClientInSock.get()); zhttpClientInValveConnection = zhttpClientInValve->readyRead.connect(boost::bind(&Wrapper::zhttpClientIn_readyRead, this, boost::placeholders::_1)); - zhttpServerInSock = new QZmq::Socket(QZmq::Socket::Pull, this); - zhttpServerInValve = new QZmq::Valve(zhttpServerInSock, this); + zhttpServerInSock = std::make_unique(QZmq::Socket::Pull); + zhttpServerInValve = std::make_unique(zhttpServerInSock.get()); zhttpServerInValveConnection = zhttpServerInValve->readyRead.connect(boost::bind(&Wrapper::zhttpServerIn_readyRead, this, boost::placeholders::_1)); - zhttpServerInStreamSock = new QZmq::Socket(QZmq::Socket::Router, this); + zhttpServerInStreamSock = std::make_unique(QZmq::Socket::Router); zhttpServerInStreamSock->setIdentity("test-server"); - zhttpServerInStreamValve = new QZmq::Valve(zhttpServerInStreamSock, this); + zhttpServerInStreamValve = std::make_unique(zhttpServerInStreamSock.get()); zhttpServerInStreamValveConnection = zhttpServerInStreamValve->readyRead.connect(boost::bind(&Wrapper::zhttpServerInStream_readyRead, this, boost::placeholders::_1)); - zhttpServerOutSock = new QZmq::Socket(QZmq::Socket::Pub, this); + zhttpServerOutSock = std::make_unique(QZmq::Socket::Pub); // handler sockets - handlerInspectSock = new QZmq::Socket(QZmq::Socket::Router, this); + handlerInspectSock = std::make_unique(QZmq::Socket::Router); - handlerAcceptSock = new QZmq::Socket(QZmq::Socket::Router, this); - handlerAcceptValve = new QZmq::Valve(handlerAcceptSock, this); + handlerAcceptSock = std::make_unique(QZmq::Socket::Router); + handlerAcceptValve = std::make_unique(handlerAcceptSock.get()); handlerAcceptValveConnection = handlerAcceptValve->readyRead.connect(boost::bind(&Wrapper::handlerAccept_readyRead, this, boost::placeholders::_1)); - handlerInspectValve = new QZmq::Valve(handlerInspectSock, this); + handlerInspectValve = std::make_unique(handlerInspectSock.get()); handlerInspectValveConnection = handlerInspectValve->readyRead.connect(boost::bind(&Wrapper::handlerInspect_readyRead, this, boost::placeholders::_1)); - handlerRetryOutSock = new QZmq::Socket(QZmq::Socket::Router, this); + handlerRetryOutSock = std::make_unique(QZmq::Socket::Router); } void startHttp() @@ -634,8 +635,11 @@ private slots: delete domainMap; delete wrapper; + // ensure deferred deletes are processed QCoreApplication::instance()->sendPostedEvents(); - RTimer::deinit(); + + Timer::deinit(); + DeferCall::cleanup(); } void passthrough() diff --git a/src/proxy/proxysession.cpp b/src/proxy/proxysession.cpp index 02d1a231b..fa1eb50be 100644 --- a/src/proxy/proxysession.cpp +++ b/src/proxy/proxysession.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2012-2023 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -25,7 +25,6 @@ #include #include -#include #include #include #include "packet/statspacket.h" @@ -745,7 +744,7 @@ class ProxySession::Private : public QObject if(!buffering && pendingWrites()) return; - QPointer self = this; + std::weak_ptr self = q->d; bool wasAllowed = addAllowed; @@ -914,7 +913,7 @@ class ProxySession::Private : public QObject if(wasAllowed && !addAllowed) { q->addNotAllowed(); - if(!self) + if(self.expired()) return; } } @@ -925,7 +924,7 @@ class ProxySession::Private : public QObject // this method emits signals void checkIncomingResponseFinished() { - QPointer self = this; + std::weak_ptr self = q->d; if(zhttpRequest->isFinished() && zhttpRequest->bytesAvailable() == 0) { @@ -946,7 +945,7 @@ class ProxySession::Private : public QObject { addAllowed = false; q->addNotAllowed(); - if(!self) + if(self.expired()) return; } @@ -1218,9 +1217,9 @@ class ProxySession::Private : public QObject if(!intReq) logFinished(si); - QPointer self = this; + std::weak_ptr self = q->d; q->requestSessionDestroyed(si->rs, false); - if(!self) + if(self.expired()) return; ZhttpRequest *req = rs->request(); @@ -1412,13 +1411,13 @@ class ProxySession::Private : public QObject sessionItems.clear(); sessionItemsBySession.clear(); - QPointer self = this; + std::weak_ptr self = q->d; foreach(RequestSession *rs, toDestroy) { q->requestSessionDestroyed(rs, true); reqSessionConnectionMap.erase(rs); delete rs; - if(!self) + if(self.expired()) return; } @@ -1489,13 +1488,10 @@ class ProxySession::Private : public QObject ProxySession::ProxySession(ZRoutes *zroutes, ZrpcManager *acceptManager, const LogUtil::Config &logConfig, StatsManager *statsManager, QObject *parent) : QObject(parent) { - d = new Private(this, zroutes, acceptManager, logConfig, statsManager); + d = std::make_shared(this, zroutes, acceptManager, logConfig, statsManager); } -ProxySession::~ProxySession() -{ - delete d; -} +ProxySession::~ProxySession() = default; void ProxySession::setRoute(const DomainMap::Entry &route) { diff --git a/src/proxy/proxysession.h b/src/proxy/proxysession.h index 58a5194f8..1d1a1fc8f 100644 --- a/src/proxy/proxysession.h +++ b/src/proxy/proxysession.h @@ -1,5 +1,6 @@ /* * Copyright (C) 2012-2022 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -73,7 +74,7 @@ class ProxySession : public QObject private: class Private; friend class Private; - Private *d; + std::shared_ptr d; }; #endif diff --git a/src/proxy/proxyutil.cpp b/src/proxy/proxyutil.cpp index ec39ee2a0..41f9d8dc0 100644 --- a/src/proxy/proxyutil.cpp +++ b/src/proxy/proxyutil.cpp @@ -177,7 +177,7 @@ void manipulateRequestHeaders(const char *logprefix, void *object, HttpRequestDa requestData->headers.removeAll("Grip-Feature"); requestData->headers += HttpHeader("Grip-Feature", - "status, session, link:next, link:gone, filter:skip-self, filter:skip-users, filter:require-sub, filter:build-id, filter:var-subst"); + "status, session, link:next, link:gone, filter:skip-self, filter:skip-users, filter:require-sub, filter:build-id, filter:var-subst, filter:http-check, filter:http-modify"); if(!idata.sid.isEmpty()) { diff --git a/src/proxy/requestsession.cpp b/src/proxy/requestsession.cpp index 7c67074b2..97073cae6 100644 --- a/src/proxy/requestsession.cpp +++ b/src/proxy/requestsession.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2012-2023 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -24,7 +24,6 @@ #include "requestsession.h" #include -#include #include #include #include @@ -36,6 +35,7 @@ #include "qtcompat.h" #include "bufferlist.h" #include "log.h" +#include "defercall.h" #include "layertracker.h" #include "sockjsmanager.h" #include "inspectdata.h" @@ -200,6 +200,7 @@ class RequestSession::Private : public QObject ZhttpReqConnections zhttpReqConnections; Connection inspectFinishedConnection; Connection acceptFinishedConnection; + DeferCall deferCall; Private(RequestSession *_q, int _workerId, DomainMap *_domainMap = 0, SockJsManager *_sockJsManager = 0, ZrpcManager *_inspectManager = 0, ZrpcChecker *_inspectChecker = 0, ZrpcManager *_acceptManager = 0, StatsManager *_stats = 0) : QObject(_q), @@ -308,7 +309,7 @@ class RequestSession::Private : public QObject isSockJs = true; sockJsManager->giveRequest(zhttpRequest, route.sockJsPath.length(), route.sockJsAsPath, route); zhttpRequest = 0; - QMetaObject::invokeMethod(this, "doFinished", Qt::QueuedConnection); + deferCall.defer([=] { doFinished(); }); return; } } @@ -480,7 +481,7 @@ class RequestSession::Private : public QObject if(!inspectRequest) { log_debug("inspect not available"); - QMetaObject::invokeMethod(this, "doInspectError", Qt::QueuedConnection); + deferCall.defer([=] { doInspectError(); }); } } } @@ -559,7 +560,7 @@ class RequestSession::Private : public QObject if(!pendingResponseUpdate) { pendingResponseUpdate = true; - QMetaObject::invokeMethod(this, "doResponseUpdate", Qt::QueuedConnection); + deferCall.defer([=] { doResponseUpdate(); }); } } @@ -811,7 +812,7 @@ class RequestSession::Private : public QObject void zhttpRequest_bytesWritten(int count) { - QPointer self = this; + std::weak_ptr self = q->d; if(!jsonpCallback.isEmpty()) { @@ -822,7 +823,7 @@ class RequestSession::Private : public QObject else q->bytesWritten(count); - if(!self) + if(self.expired()) return; if(zhttpRequest->isFinished()) @@ -973,7 +974,6 @@ class RequestSession::Private : public QObject } } -public slots: void doResponseUpdate() { pendingResponseUpdate = false; @@ -1190,13 +1190,10 @@ public slots: RequestSession::RequestSession(int workerId, DomainMap *domainMap, SockJsManager *sockJsManager, ZrpcManager *inspectManager, ZrpcChecker *inspectChecker, ZrpcManager *acceptManager, StatsManager *stats, QObject *parent) : QObject(parent) { - d = new Private(this, workerId, domainMap, sockJsManager, inspectManager, inspectChecker, acceptManager, stats); + d = std::make_shared(this, workerId, domainMap, sockJsManager, inspectManager, inspectChecker, acceptManager, stats); } -RequestSession::~RequestSession() -{ - delete d; -} +RequestSession::~RequestSession() = default; bool RequestSession::isRetry() const { diff --git a/src/proxy/requestsession.h b/src/proxy/requestsession.h index 9ec3da653..c25f39eb9 100644 --- a/src/proxy/requestsession.h +++ b/src/proxy/requestsession.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2012-2023 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -120,7 +120,7 @@ class RequestSession : public QObject private: class Private; friend class Private; - Private *d; + std::shared_ptr d; }; #endif diff --git a/src/proxy/sockjsmanager.cpp b/src/proxy/sockjsmanager.cpp index dd2d0ec8e..66a034f09 100644 --- a/src/proxy/sockjsmanager.cpp +++ b/src/proxy/sockjsmanager.cpp @@ -34,7 +34,7 @@ #include "qtcompat.h" #include "log.h" #include "bufferlist.h" -#include "rtimer.h" +#include "timer.h" #include "zhttprequest.h" #include "zwebsocket.h" #include "sockjssession.h" @@ -99,7 +99,7 @@ class SockJsManager::Private : public QObject QByteArray lastPart; bool pending; SockJsSession *ext; - std::unique_ptr timer; + std::unique_ptr timer; QVariant closeValue; Connection timerConnection; @@ -141,7 +141,7 @@ class SockJsManager::Private : public QObject QHash sessionsBySocket; QHash sessionsById; QHash sessionsByExt; - QHash sessionsByTimer; + QHash sessionsByTimer; QList pendingSessions; QByteArray iframeHtml; QByteArray iframeHtmlEtag; @@ -199,7 +199,7 @@ class SockJsManager::Private : public QObject if(s->closeValue.isValid()) { // if there's a close value, hang around for a little bit - s->timer = std::make_unique(); + s->timer = std::make_unique(); s->timerConnection = s->timer->timeout.connect(boost::bind(&Private::timer_timeout, this, s->timer.get())); s->timer->setSingleShot(true); sessionsByTimer.insert(s->timer.get(), s); @@ -674,7 +674,7 @@ class SockJsManager::Private : public QObject } private: - void timer_timeout(RTimer *timer) + void timer_timeout(Timer *timer) { Session *s = sessionsByTimer.value(timer); assert(s); diff --git a/src/proxy/sockjssession.cpp b/src/proxy/sockjssession.cpp index 3db6ba913..78e9bc231 100644 --- a/src/proxy/sockjssession.cpp +++ b/src/proxy/sockjssession.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2015-2021 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -24,7 +24,6 @@ #include "sockjssession.h" #include -#include #include #include #include @@ -33,7 +32,8 @@ #include "log.h" #include "bufferlist.h" #include "packet/httprequestdata.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "zhttprequest.h" #include "zwebsocket.h" #include "sockjsmanager.h" @@ -155,7 +155,7 @@ class SockJsSession::Private : public QObject int pendingWrittenBytes; QList pendingWrites; QHash requests; - std::unique_ptr keepAliveTimer; + std::unique_ptr keepAliveTimer; int closeCode; QString closeReason; bool closeSent; @@ -166,6 +166,7 @@ class SockJsSession::Private : public QObject map reqConnectionMap; WSConnections wsConnection; Connection keepAliveTimerConnection; + DeferCall deferCall; Private(SockJsSession *_q) : QObject(_q), @@ -187,7 +188,7 @@ class SockJsSession::Private : public QObject peerCloseCode(-1), updating(false) { - keepAliveTimer = std::make_unique(); + keepAliveTimer = std::make_unique(); keepAliveTimerConnection = keepAliveTimer->timeout.connect(boost::bind(&Private::keepAliveTimer_timeout, this)); } @@ -589,7 +590,7 @@ class SockJsSession::Private : public QObject state = Idle; applyLinger(); cleanup(); - QMetaObject::invokeMethod(this, "doClosed", Qt::QueuedConnection); + deferCall.defer([=] { doClosed(); }); } else tryWrite(); @@ -650,9 +651,9 @@ class SockJsSession::Private : public QObject if(bytes > 0) { - QPointer self = this; + std::weak_ptr self = q->d; q->writeBytesChanged(); - if(!self) + if(self.expired()) return; } @@ -678,7 +679,7 @@ class SockJsSession::Private : public QObject bool tryRead() { - QPointer self = this; + std::weak_ptr self = q->d; if(mode == Http) { @@ -722,7 +723,7 @@ class SockJsSession::Private : public QObject if(emitReadyRead) { q->readyRead(); - if(!self) + if(self.expired()) return false; } } @@ -837,7 +838,7 @@ class SockJsSession::Private : public QObject if(emitReadyRead) { q->readyRead(); - if(!self) + if(self.expired()) return false; } } @@ -850,7 +851,7 @@ class SockJsSession::Private : public QObject if(!updating) { updating = true; - QMetaObject::invokeMethod(this, "doUpdate", Qt::QueuedConnection); + deferCall.defer([=] { doUpdate(); }); } } @@ -1044,7 +1045,6 @@ class SockJsSession::Private : public QObject q->error(); } -private slots: void doUpdate() { updating = false; @@ -1115,13 +1115,10 @@ private slots: SockJsSession::SockJsSession(QObject *parent) : WebSocket(parent) { - d = new Private(this); + d = std::make_shared(this); } -SockJsSession::~SockJsSession() -{ - delete d; -} +SockJsSession::~SockJsSession() = default; QByteArray SockJsSession::sid() const { diff --git a/src/proxy/sockjssession.h b/src/proxy/sockjssession.h index 509b250a8..9d4645197 100644 --- a/src/proxy/sockjssession.h +++ b/src/proxy/sockjssession.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2015 Fanout, Inc. - * Copyright (C) 2023 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -83,7 +83,7 @@ class SockJsSession : public WebSocket private: class Private; friend class Private; - Private *d; + std::shared_ptr d; friend class SockJsManager; SockJsSession(QObject *parent = 0); diff --git a/src/proxy/testhttprequest.cpp b/src/proxy/testhttprequest.cpp index dfe363087..fa11dcea6 100644 --- a/src/proxy/testhttprequest.cpp +++ b/src/proxy/testhttprequest.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2016 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -25,6 +26,7 @@ #include #include #include "log.h" +#include "defercall.h" #include "bufferlist.h" #include "packet/httprequestdata.h" #include "packet/httpresponsedata.h" @@ -53,6 +55,7 @@ class TestHttpRequest::Private : public QObject bool requestBodyFinished; BufferList responseBody; ErrorCondition errorCondition; + DeferCall deferCall; Private(TestHttpRequest *_q) : QObject(_q), @@ -63,7 +66,6 @@ class TestHttpRequest::Private : public QObject { } -public slots: void doBytesWritten(int cnt){ q->bytesWritten(cnt); } @@ -181,6 +183,11 @@ void TestHttpRequest::setIgnoreTlsErrors(bool on) Q_UNUSED(on); } +void TestHttpRequest::setTimeout(int msecs) +{ + Q_UNUSED(msecs); +} + void TestHttpRequest::start(const QString &method, const QUrl &uri, const HttpHeaders &headers) { assert(d->state == Private::Idle); @@ -209,7 +216,7 @@ void TestHttpRequest::writeBody(const QByteArray &body) if(d->requestBody.size() + body.size() > MAX_REQUEST_SIZE) { d->state = Private::Responding; - QMetaObject::invokeMethod(d, "processRequest", Qt::QueuedConnection); + d->deferCall.defer([=] { d->processRequest(); }); return; } @@ -219,7 +226,8 @@ void TestHttpRequest::writeBody(const QByteArray &body) { d->requestBody += buf; - QMetaObject::invokeMethod(this, "doBytesWritten", Qt::QueuedConnection, Q_ARG(int, buf.size())); + int written = buf.size(); + d->deferCall.defer([=] { d->doBytesWritten(written); }); } } } @@ -231,7 +239,7 @@ void TestHttpRequest::endBody() d->requestBodyFinished = true; d->state = Private::Responding; - QMetaObject::invokeMethod(d, "processRequest", Qt::QueuedConnection); + d->deferCall.defer([=] { d->processRequest(); }); } } diff --git a/src/proxy/testhttprequest.h b/src/proxy/testhttprequest.h index a9873d8e4..6b20726a1 100644 --- a/src/proxy/testhttprequest.h +++ b/src/proxy/testhttprequest.h @@ -45,6 +45,7 @@ class TestHttpRequest : public HttpRequest virtual void setIgnorePolicies(bool on); virtual void setTrustConnectHost(bool on); virtual void setIgnoreTlsErrors(bool on); + virtual void setTimeout(int msecs); virtual void start(const QString &method, const QUrl &uri, const HttpHeaders &headers); virtual void beginResponse(int code, const QByteArray &reason, const HttpHeaders &headers); diff --git a/src/proxy/testwebsocket.cpp b/src/proxy/testwebsocket.cpp index 417a32585..25365a28a 100644 --- a/src/proxy/testwebsocket.cpp +++ b/src/proxy/testwebsocket.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2016 Fanout, Inc. - * Copyright (C) 2023 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -27,6 +27,7 @@ #include #include #include +#include "defercall.h" #include "packet/httprequestdata.h" #include "packet/httpresponsedata.h" #include "statusreasons.h" @@ -55,6 +56,7 @@ class TestWebSocket::Private : public QObject int peerCloseCode; QString peerCloseReason; ErrorCondition errorCondition; + DeferCall deferCall; Private(TestWebSocket *_q) : QObject(_q), @@ -66,7 +68,6 @@ class TestWebSocket::Private : public QObject { } -public slots: void handleConnect() { QString path = request.uri.path(); @@ -118,7 +119,7 @@ public slots: q->connected(); if(gripEnabled && !channels.isEmpty()) - QMetaObject::invokeMethod(this, "doReadyRead", Qt::QueuedConnection); + deferCall.defer([=] { doReadyRead(); }); } else { @@ -203,7 +204,7 @@ void TestWebSocket::start(const QUrl &uri, const HttpHeaders &headers) d->state = Private::Connecting; - QMetaObject::invokeMethod(d, "handleConnect", Qt::QueuedConnection); + d->deferCall.defer([=] { d->handleConnect(); }); } void TestWebSocket::respondSuccess(const QByteArray &reason, const HttpHeaders &headers) @@ -311,13 +312,14 @@ void TestWebSocket::writeFrame(const Frame &frame) d->inFrames += tmp; - QMetaObject::invokeMethod(d, "doFramesWritten", Qt::QueuedConnection, Q_ARG(int, 1), Q_ARG(int, tmp.data.size())); - QMetaObject::invokeMethod(d, "doReadyRead", Qt::QueuedConnection); + int contentBytesWritten = tmp.data.size(); + d->deferCall.defer([=] { d->doFramesWritten(1, contentBytesWritten); }); + d->deferCall.defer([=] { d->doReadyRead(); }); } WebSocket::Frame TestWebSocket::readFrame() { - QMetaObject::invokeMethod(d, "doWriteBytesChanged", Qt::QueuedConnection); + d->deferCall.defer([=] { d->doWriteBytesChanged(); }); return d->inFrames.takeFirst(); } @@ -328,7 +330,7 @@ void TestWebSocket::close(int code, const QString &reason) d->peerCloseCode = code; d->peerCloseReason = reason; - QMetaObject::invokeMethod(d, "handleClose", Qt::QueuedConnection); + d->deferCall.defer([=] { d->handleClose(); }); } #include "testwebsocket.moc" diff --git a/src/proxy/updater.cpp b/src/proxy/updater.cpp index 203fabb65..f2ae2f1ad 100644 --- a/src/proxy/updater.cpp +++ b/src/proxy/updater.cpp @@ -32,7 +32,7 @@ #include #include "qtcompat.h" #include "log.h" -#include "rtimer.h" +#include "timer.h" #include "httpheaders.h" #include "zhttpmanager.h" #include "zhttprequest.h" @@ -83,7 +83,7 @@ class Updater::Private : public QObject QString currentVersion; QString org; ZhttpManager *zhttpManager; - std::unique_ptr timer; + std::unique_ptr timer; ZhttpRequest *req; Report report; QDateTime lastLogTime; @@ -100,7 +100,7 @@ class Updater::Private : public QObject zhttpManager(zhttp), req(0) { - timer = std::make_unique(); + timer = std::make_unique(); timerConnection = timer->timeout.connect(boost::bind(&Private::timer_timeout, this)); timer->setInterval(mode == ReportMode ? REPORT_INTERVAL : CHECK_INTERVAL); timer->start(); diff --git a/src/proxy/websocketoverhttp.cpp b/src/proxy/websocketoverhttp.cpp index d2c6abe38..7c456a205 100644 --- a/src/proxy/websocketoverhttp.cpp +++ b/src/proxy/websocketoverhttp.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2022 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -24,7 +24,6 @@ #include "websocketoverhttp.h" #include -#include #include #include "log.h" #include "bufferlist.h" @@ -33,7 +32,8 @@ #include "zhttprequest.h" #include "zhttpmanager.h" #include "uuidutil.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #define BUFFER_SIZE 200000 #define FRAME_SIZE_MAX 16384 @@ -233,13 +233,14 @@ class WebSocketOverHttp::Private : public QObject bool disconnecting; bool disconnectSent; bool updateQueued; - std::unique_ptr keepAliveTimer; - std::unique_ptr retryTimer; + std::unique_ptr keepAliveTimer; + std::unique_ptr retryTimer; int retries; int maxEvents; ReqConnections reqConnections; Connection keepAliveTimerConnection; Connection retryTimerConnection; + DeferCall deferCall; Private(WebSocketOverHttp *_q) : QObject(_q), @@ -271,11 +272,11 @@ class WebSocketOverHttp::Private : public QObject if(!g_disconnectManager) g_disconnectManager = new DisconnectManager; - keepAliveTimer = std::make_unique(); + keepAliveTimer = std::make_unique(); keepAliveTimerConnection = keepAliveTimer->timeout.connect(boost::bind(&Private::keepAliveTimer_timeout, this)); keepAliveTimer->setSingleShot(true); - retryTimer = std::make_unique(); + retryTimer = std::make_unique(); retryTimerConnection = retryTimer->timeout.connect(boost::bind(&Private::retryTimer_timeout, this)); retryTimer->setSingleShot(true); } @@ -475,7 +476,7 @@ class WebSocketOverHttp::Private : public QObject if((int)pendingErrorCondition == -1) { pendingErrorCondition = e; - QMetaObject::invokeMethod(this, "doError", Qt::QueuedConnection); + deferCall.defer([=] { doError(); }); } } @@ -777,7 +778,7 @@ class WebSocketOverHttp::Private : public QObject return; } - QPointer self = this; + std::weak_ptr self = q->d; bool emitConnected = false; bool emitReadyRead = false; @@ -841,21 +842,21 @@ class WebSocketOverHttp::Private : public QObject if(emitConnected) { q->connected(); - if(!self) + if(self.expired()) return; } if(emitReadyRead) { q->readyRead(); - if(!self) + if(self.expired()) return; } if(reqFrames > 0) { q->framesWritten(reqFrames, reqContentSize); - if(!self) + if(self.expired()) return; } @@ -867,7 +868,7 @@ class WebSocketOverHttp::Private : public QObject if(hadContent) { q->writeBytesChanged(); - if(!self) + if(self.expired()) return; } @@ -984,7 +985,6 @@ class WebSocketOverHttp::Private : public QObject q->error(); } -private slots: void keepAliveTimer_timeout() { update(); @@ -1007,13 +1007,12 @@ private slots: WebSocketOverHttp::WebSocketOverHttp(ZhttpManager *zhttpManager, QObject *parent) : WebSocket(parent) { - d = new Private(this); + d = std::make_shared(this); d->zhttpManager = zhttpManager; } WebSocketOverHttp::WebSocketOverHttp(QObject *parent) : - WebSocket(parent), - d(0) + WebSocket(parent) { } @@ -1026,11 +1025,9 @@ WebSocketOverHttp::~WebSocketOverHttp() sock->d = d; d->setParent(sock); d->q = sock; - d = 0; + d.reset(); g_disconnectManager->addSocket(sock); } - - delete d; } void WebSocketOverHttp::setConnectionId(const QByteArray &id) diff --git a/src/proxy/websocketoverhttp.h b/src/proxy/websocketoverhttp.h index 4b6d4923b..53af2b161 100644 --- a/src/proxy/websocketoverhttp.h +++ b/src/proxy/websocketoverhttp.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2020 Fanout, Inc. - * Copyright (C) 2023-2024 Fastly, Inc. + * Copyright (C) 2023-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -96,7 +96,7 @@ class WebSocketOverHttp : public WebSocket class Private; friend class Private; - Private *d; + std::shared_ptr d; static thread_local DisconnectManager *g_disconnectManager; static thread_local int g_maxManagedDisconnects; diff --git a/src/proxy/wscontrolmanager.cpp b/src/proxy/wscontrolmanager.cpp index 906256315..df4320b22 100644 --- a/src/proxy/wscontrolmanager.cpp +++ b/src/proxy/wscontrolmanager.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2020 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -24,14 +24,13 @@ #include "wscontrolmanager.h" #include -#include #include #include #include "qzmqsocket.h" #include "qzmqvalve.h" #include "qzmqreqmessage.h" #include "log.h" -#include "rtimer.h" +#include "timer.h" #include "tnetstring.h" #include "zutil.h" #include "logutil.h" @@ -68,11 +67,11 @@ class WsControlManager::Private : public QObject int ipcFileMode; QStringList initSpecs; QStringList streamSpecs; - QZmq::Socket *initSock; - QZmq::Socket *streamSock; - QZmq::Valve *streamValve; + std::unique_ptr initSock; + std::unique_ptr streamSock; + std::unique_ptr streamValve; QHash sessionsByCid; - std::unique_ptr refreshTimer; + std::unique_ptr refreshTimer; QHash keepAliveRegistrations; QMap, KeepAliveRegistration*> sessionsByLastRefresh; QSet sessionRefreshBuckets[SESSION_REFRESH_BUCKETS]; @@ -84,12 +83,9 @@ class WsControlManager::Private : public QObject QObject(_q), q(_q), ipcFileMode(-1), - initSock(0), - streamSock(0), - streamValve(0), currentSessionRefreshBucket(0) { - refreshTimer = std::make_unique(); + refreshTimer = std::make_unique(); refreshTimerConnection = refreshTimer->timeout.connect(boost::bind(&Private::refresh_timeout, this)); } @@ -101,9 +97,9 @@ class WsControlManager::Private : public QObject bool setupInit() { - delete initSock; + initSock.reset(); - initSock = new QZmq::Socket(QZmq::Socket::Push, this); + initSock = std::make_unique(QZmq::Socket::Push); initSock->setHwm(DEFAULT_HWM); initSock->setShutdownWaitTime(0); @@ -111,7 +107,7 @@ class WsControlManager::Private : public QObject foreach(const QString &spec, initSpecs) { QString errorMessage; - if(!ZUtil::setupSocket(initSock, spec, true, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(initSock.get(), spec, true, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; @@ -123,9 +119,10 @@ class WsControlManager::Private : public QObject bool setupStream() { - delete streamSock; + streamValve.reset(); + streamSock.reset(); - streamSock = new QZmq::Socket(QZmq::Socket::Router, this); + streamSock = std::make_unique(QZmq::Socket::Router); streamSock->setIdentity(identity); streamSock->setHwm(DEFAULT_HWM); @@ -134,14 +131,14 @@ class WsControlManager::Private : public QObject foreach(const QString &spec, streamSpecs) { QString errorMessage; - if(!ZUtil::setupSocket(streamSock, spec, true, ipcFileMode, &errorMessage)) + if(!ZUtil::setupSocket(streamSock.get(), spec, true, ipcFileMode, &errorMessage)) { log_error("%s", qPrintable(errorMessage)); return false; } } - streamValve = new QZmq::Valve(streamSock, this); + streamValve = std::make_unique(streamSock.get()); streamValveConnection = streamValve->readyRead.connect(boost::bind(&Private::stream_readyRead, this, boost::placeholders::_1)); streamValve->open(); @@ -291,7 +288,7 @@ class WsControlManager::Private : public QObject return; } - QPointer self = this; + std::weak_ptr self = q->d; foreach(const WsControlPacket::Item &i, p.items) { @@ -314,12 +311,11 @@ class WsControlManager::Private : public QObject s->handle(p.from, i); - if(!self) + if(self.expired()) return; } } -private slots: void refresh_timeout() { qint64 now = QDateTime::currentMSecsSinceEpoch(); @@ -425,7 +421,7 @@ private slots: WsControlManager::WsControlManager() { - d = std::make_unique(this); + d = std::make_shared(this); } WsControlManager::~WsControlManager() = default; diff --git a/src/proxy/wscontrolmanager.h b/src/proxy/wscontrolmanager.h index 70a5e68f6..d0dee72d2 100644 --- a/src/proxy/wscontrolmanager.h +++ b/src/proxy/wscontrolmanager.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2014 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -48,7 +48,7 @@ class WsControlManager : public QObject private: class Private; - std::unique_ptr d; + std::shared_ptr d; friend class WsControlSession; void link(WsControlSession *s, const QByteArray &cid); diff --git a/src/proxy/wscontrolsession.cpp b/src/proxy/wscontrolsession.cpp index 0fc108ab7..9f24a4485 100644 --- a/src/proxy/wscontrolsession.cpp +++ b/src/proxy/wscontrolsession.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2022 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -27,7 +27,7 @@ #include #include #include -#include "rtimer.h" +#include "timer.h" #include "wscontrolmanager.h" #define SESSION_TTL 60 @@ -46,14 +46,16 @@ class WsControlSession::Private : public QObject QList pendingItems; QHash pendingRequests; QList pendingSendEventWrites; - std::unique_ptr requestTimer; + std::unique_ptr requestTimer; QByteArray peer; QByteArray cid; + bool debug; QByteArray route; bool separateStats; QByteArray channelPrefix; int logLevel; QUrl uri; + bool targetTrusted; Connection requestTimerConnection; Private(WsControlSession *_q) : @@ -61,10 +63,12 @@ class WsControlSession::Private : public QObject q(_q), manager(0), nextReqId(0), + debug(false), separateStats(false), - logLevel(-1) + logLevel(-1), + targetTrusted(false) { - requestTimer = std::make_unique(); + requestTimer = std::make_unique(); requestTimerConnection = requestTimer->timeout.connect(boost::bind(&Private::requestTimer_timeout, this)); requestTimer->setSingleShot(true); } @@ -98,11 +102,13 @@ class WsControlSession::Private : public QObject WsControlPacket::Item i; i.type = WsControlPacket::Item::Here; i.requestId = QByteArray::number(reqId); + i.debug = debug; i.route = route; i.separateStats = separateStats; i.channelPrefix = channelPrefix; i.logLevel = logLevel; i.uri = uri; + i.trusted = targetTrusted; i.ttl = SESSION_TTL; write(i, true); } @@ -298,7 +304,6 @@ class WsControlSession::Private : public QObject } } -private slots: void requestTimer_timeout() { // on error, destroy any other pending requests @@ -326,13 +331,15 @@ QByteArray WsControlSession::cid() const return d->cid; } -void WsControlSession::start(const QByteArray &routeId, bool separateStats, const QByteArray &channelPrefix, int logLevel, const QUrl &uri) +void WsControlSession::start(bool debug, const QByteArray &routeId, bool separateStats, const QByteArray &channelPrefix, int logLevel, const QUrl &uri, bool targetTrusted) { + d->debug = debug; d->route = routeId; d->separateStats = separateStats; d->channelPrefix = channelPrefix; d->logLevel = logLevel; d->uri = uri; + d->targetTrusted = targetTrusted; d->start(); } diff --git a/src/proxy/wscontrolsession.h b/src/proxy/wscontrolsession.h index 1c1a933e7..1ca033648 100644 --- a/src/proxy/wscontrolsession.h +++ b/src/proxy/wscontrolsession.h @@ -45,7 +45,7 @@ class WsControlSession : public QObject QByteArray peer() const; QByteArray cid() const; - void start(const QByteArray &routeId, bool separateStats, const QByteArray &channelPrefix, int logLevel, const QUrl &uri); + void start(bool debug, const QByteArray &routeId, bool separateStats, const QByteArray &channelPrefix, int logLevel, const QUrl &uri, bool targetTrusted); void sendGripMessage(const QByteArray &message); void sendNeedKeepAlive(); void sendSubscribe(const QByteArray &channel); diff --git a/src/proxy/wsproxysession.cpp b/src/proxy/wsproxysession.cpp index aa659e03e..427fa60b2 100644 --- a/src/proxy/wsproxysession.cpp +++ b/src/proxy/wsproxysession.cpp @@ -1,6 +1,6 @@ /* * Copyright (C) 2014-2023 Fanout, Inc. - * Copyright (C) 2024 Fastly, Inc. + * Copyright (C) 2024-2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -32,7 +32,8 @@ #include #include "packet/httprequestdata.h" #include "log.h" -#include "rtimer.h" +#include "timer.h" +#include "defercall.h" #include "jwt.h" #include "zhttpmanager.h" #include "zwebsocket.h" @@ -302,7 +303,7 @@ class WsProxySession::Private : public QObject bool detached; QDateTime activityTime; QByteArray publicCid; - RTimer *keepAliveTimer; + Timer *keepAliveTimer; WsControl::KeepAliveMode keepAliveMode; int keepAliveTimeout; QList queuedInFrames; // frames to deliver after out read finishes @@ -386,7 +387,7 @@ class WsProxySession::Private : public QObject { keepAliveConnection.disconnect(); keepAliveTimer->setParent(0); - keepAliveTimer->deleteLater(); + DeferCall::deleteLater(keepAliveTimer); keepAliveTimer = 0; } } @@ -825,7 +826,6 @@ class WsProxySession::Private : public QObject statsManager->incCounter(route.statsRoute(), c, count); } -private slots: void in_readyRead() { if((outSock && outSock->state() == WebSocket::Connected) || detached) @@ -949,7 +949,7 @@ private slots: wsControl->cancelEventReceived.connect(boost::bind(&Private::wsControl_cancelEventReceived, this)), wsControl->error.connect(boost::bind(&Private::wsControl_error, this)) }; - wsControl->start(route.id, route.separateStats, channelPrefix, route.logLevel, inSock->requestUri()); + wsControl->start(route.debug, route.id, route.separateStats, channelPrefix, route.logLevel, inSock->requestUri(), target.trusted); foreach(const QString &subChannel, target.subscriptions) { @@ -1109,7 +1109,7 @@ private slots: if(!keepAliveTimer) { - keepAliveTimer = new RTimer; + keepAliveTimer = new Timer; keepAliveConnection = keepAliveTimer->timeout.connect(boost::bind(&Private::keepAliveTimer_timeout, this)); keepAliveTimer->setSingleShot(true); } @@ -1124,7 +1124,7 @@ private slots: void wsControl_refreshEventReceived() { - WebSocketOverHttp *woh = qobject_cast(outSock); + WebSocketOverHttp *woh = dynamic_cast(outSock); if(woh) woh->refresh(); } diff --git a/src/proxy/zroutes.cpp b/src/proxy/zroutes.cpp index c910cf687..cf406841a 100644 --- a/src/proxy/zroutes.cpp +++ b/src/proxy/zroutes.cpp @@ -27,7 +27,7 @@ #include #include #include "log.h" -#include "rtimer.h" +#include "timer.h" using Connection = boost::signals2::scoped_connection; @@ -96,7 +96,7 @@ class ZRoutes::Private : public QObject Item *defaultItem; QHash itemsBySpec; QHash itemsByManager; - std::unique_ptr cleanupTimer; + std::unique_ptr cleanupTimer; Connection cleanupTimerConnection; Private(ZRoutes *_q) : @@ -104,7 +104,7 @@ class ZRoutes::Private : public QObject q(_q), defaultItem(0) { - cleanupTimer = std::make_unique(); + cleanupTimer = std::make_unique(); cleanupTimerConnection = cleanupTimer->timeout.connect(boost::bind(&Private::removeUnused, this)); cleanupTimer->setInterval(10000); cleanupTimer->start(); @@ -180,7 +180,6 @@ class ZRoutes::Private : public QObject delete i; } -public slots: void removeUnused() { QList toRemove; diff --git a/src/proxy/zrpcchecker.cpp b/src/proxy/zrpcchecker.cpp index c9da7e297..b76e0b63b 100644 --- a/src/proxy/zrpcchecker.cpp +++ b/src/proxy/zrpcchecker.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2015 Fanout, Inc. + * Copyright (C) 2025 Fastly, Inc. * * This file is part of Pushpin. * @@ -24,7 +25,7 @@ #include #include -#include "rtimer.h" +#include "timer.h" #include "zrpcrequest.h" #define CHECK_TIMEOUT 8 @@ -62,7 +63,7 @@ class ZrpcChecker::Private : public QObject ZrpcChecker *q; bool avail; - std::unique_ptr timer; + std::unique_ptr timer; QHash requestsByReq; map reqConnectionMap; Connection timerConnection; @@ -72,7 +73,7 @@ class ZrpcChecker::Private : public QObject q(_q), avail(true) { - timer = std::make_unique(); + timer = std::make_unique(); timerConnection = timer->timeout.connect(boost::bind(&Private::timer_timeout, this)); timer->setSingleShot(true); } @@ -212,7 +213,6 @@ class ZrpcChecker::Private : public QObject delete i; } -public slots: void timer_timeout() { avail = false; diff --git a/src/runner/service.rs b/src/runner/service.rs index bc1eddea4..947b35009 100644 --- a/src/runner/service.rs +++ b/src/runner/service.rs @@ -434,6 +434,41 @@ fn start_log_handler( result } +fn log_message(name: &str, level: log::Level, msg: &str) { + const MAX_MSG_LEN: usize = 2048; // Set your desired message length limit + + // Find the position of the 3rd space (' ') in the string + let index = msg + .char_indices() + .filter(|&(_, c)| c == ' ') + .nth(2) + .map(|(i, _)| i) + .unwrap_or_else(|| 0); + + // Truncate the message to MAX_MSG_LEN + let truncated_msg = if msg.len() > MAX_MSG_LEN { + let split_len = MAX_MSG_LEN / 2; + let head: String = msg.chars().take(split_len).collect(); + let tail: String = msg.chars().rev().take(split_len).collect::().chars().rev().collect(); + format!("{}...{}", head, tail) + } else { + msg.to_string() + }; + + log::logger().log( + &log::Record::builder() + .level(level) + .target(name) + .args(format_args!( + "{} [{}]{}", + &truncated_msg[..index.min(truncated_msg.len())], // Prevent out-of-bounds access + name, + &truncated_msg[index.min(truncated_msg.len())..] + )) + .build(), + ); +} +/* fn log_message(name: &str, level: log::Level, msg: &str) { // Find the position of the 3rd space (' ') in the string let index = msg @@ -456,3 +491,4 @@ fn log_message(name: &str, level: log::Level, msg: &str) { .build(), ); } +*/ diff --git a/tools/command.py b/tools/command.py index 8a604669c..72b546f99 100644 --- a/tools/command.py +++ b/tools/command.py @@ -4,6 +4,7 @@ import tnetstring import zmq + def make_tnet_compat(obj): if isinstance(obj, dict): out = {} @@ -16,10 +17,11 @@ def make_tnet_compat(obj): out.append(make_tnet_compat(v)) return out elif isinstance(obj, str): - return obj.encode('utf-8') + return obj.encode("utf-8") else: return obj + ctx = zmq.Context() sock = ctx.socket(zmq.REQ) sock.connect(sys.argv[1]) @@ -28,26 +30,26 @@ def make_tnet_compat(obj): if len(sys.argv) > 3: args = json.loads(sys.argv[3]) - assert(isinstance(args, dict)) + assert isinstance(args, dict) else: args = {} -print('calling {}: args={}'.format(method, repr(args))) +print("calling {}: args={}".format(method, repr(args))) req = { - b'id': str(uuid.uuid4()).encode('utf-8'), - b'method': method.encode('utf-8'), - b'args': make_tnet_compat(args) + b"id": str(uuid.uuid4()).encode("utf-8"), + b"method": method.encode("utf-8"), + b"args": make_tnet_compat(args), } sock.send(tnetstring.dumps(req)) resp = tnetstring.loads(sock.recv()) -if resp[b'success']: - value = resp[b'value'] - print('success: {}'.format(repr(value))) +if resp[b"success"]: + value = resp[b"value"] + print("success: {}".format(repr(value))) else: - condition = resp[b'condition'].decode('utf-8') - value = resp.get(b'value') - print('error: {} {}'.format(condition, repr(value))) + condition = resp[b"condition"].decode("utf-8") + value = resp.get(b"value") + print("error: {} {}".format(condition, repr(value))) diff --git a/tools/edgebroker.py b/tools/edgebroker.py index a888a8a9e..9f6a83eaa 100644 --- a/tools/edgebroker.py +++ b/tools/edgebroker.py @@ -2,8 +2,8 @@ import zmq if len(sys.argv) < 3: - print 'usage: %s [pub_spec] [sub_spec,sub_spec,...]' % sys.argv[0] - sys.exit(1) + print("usage: %s [pub_spec] [sub_spec,sub_spec,...]" % sys.argv[0]) + sys.exit(1) pub_spec = sys.argv[1] sub_specs = sys.argv[2:] @@ -12,7 +12,7 @@ sub_sock = zmq_context.socket(zmq.SUB) for spec in sub_specs: - sub_sock.connect(spec) + sub_sock.connect(spec) pub_sock = zmq_context.socket(zmq.XPUB) pub_sock.connect(pub_spec) @@ -22,17 +22,17 @@ poller.register(pub_sock, zmq.POLLIN) while True: - socks = dict(poller.poll()) - if socks.get(sub_sock) == zmq.POLLIN: - m = sub_sock.recv_multipart() - pub_sock.send_multipart(m) - elif socks.get(pub_sock) == zmq.POLLIN: - m = pub_sock.recv() - mtype = m[0] - topic = m[1:] - if mtype == '\x01': - print 'subscribing [%s]' % topic - sub_sock.setsockopt(zmq.SUBSCRIBE, topic) - elif mtype == '\x00': - print 'unsubscribing [%s]' % topic - sub_sock.setsockopt(zmq.UNSUBSCRIBE, topic) + socks = dict(poller.poll()) + if socks.get(sub_sock) == zmq.POLLIN: + m = sub_sock.recv_multipart() + pub_sock.send_multipart(m) + elif socks.get(pub_sock) == zmq.POLLIN: + m = pub_sock.recv() + mtype = m[0] + topic = m[1:] + if mtype == "\x01": + print("subscribing [%s]" % topic) + sub_sock.setsockopt(zmq.SUBSCRIBE, topic) + elif mtype == "\x00": + print("unsubscribing [%s]" % topic) + sub_sock.setsockopt(zmq.UNSUBSCRIBE, topic) diff --git a/tools/getzmquris.py b/tools/getzmquris.py index 998e30435..b8fd0d5b6 100644 --- a/tools/getzmquris.py +++ b/tools/getzmquris.py @@ -4,37 +4,39 @@ command_host = None + def resolve(uri): - if uri.startswith('tcp://'): - at = uri.find(':', 6) - addr = uri[6:at] - if addr == '*': - if command_host: - return uri[0:6] + command_host + uri[at:] - else: - return uri[0:6] + 'localhost' + uri[at:] - return uri + if uri.startswith("tcp://"): + at = uri.find(":", 6) + addr = uri[6:at] + if addr == "*": + if command_host: + return uri[0:6] + command_host + uri[at:] + else: + return uri[0:6] + "localhost" + uri[at:] + return uri + command_uri = sys.argv[1] -if command_uri.startswith('tcp://'): - at = command_uri.find(':', 6) - command_host = command_uri[6:at] +if command_uri.startswith("tcp://"): + at = command_uri.find(":", 6) + command_host = command_uri[6:at] sock = zmq.Context.instance().socket(zmq.REQ) sock.connect(command_uri) -req = {'method': 'get-zmq-uris'} +req = {"method": "get-zmq-uris"} sock.send(tnetstring.dumps(req)) resp = tnetstring.loads(sock.recv()) -if not resp.get('success'): - raise ValueError('request failed: %s' % resp) +if not resp.get("success"): + raise ValueError("request failed: %s" % resp) -v = resp['value'] +v = resp["value"] -if 'command' in v: - print 'command: %s' % resolve(v['command']) -if 'publish-pull' in v: - print 'publish-pull: %s' % resolve(v['publish-pull']) -if 'publish-sub' in v: - print 'publish-sub: %s' % resolve(v['publish-sub']) +if "command" in v: + print("command: %s" % resolve(v["command"])) +if "publish-pull" in v: + print("publish-pull: %s" % resolve(v["publish-pull"])) +if "publish-sub" in v: + print("publish-sub: %s" % resolve(v["publish-sub"])) diff --git a/tools/gripstate.py b/tools/gripstate.py index 49d87411f..e2a6f9705 100644 --- a/tools/gripstate.py +++ b/tools/gripstate.py @@ -6,108 +6,130 @@ sock = ctx.socket(zmq.REP) sock.connect(sys.argv[1]) + class Rule(object): - def __init__(self): - self.domain = None - self.path_prefix = None - self.sid_ptr = None - self.json_param = None + def __init__(self): + self.domain = None + self.path_prefix = None + self.sid_ptr = None + self.json_param = None + class Session(object): - def __init__(self): - self.last_ids = dict() + def __init__(self): + self.last_ids = dict() + rules = list() sessions = dict() + def session_detect_rules_set(new_rules): - for nr in new_rules: - found = False - for r in rules: - if r.domain == nr.domain and r.path_prefix == nr.path_prefix and r.sid_ptr == nr.sid_ptr and r.json_param == nr.json_param: - found = True - break - if found: - continue + for nr in new_rules: + found = False + for r in rules: + if ( + r.domain == nr.domain + and r.path_prefix == nr.path_prefix + and r.sid_ptr == nr.sid_ptr + and r.json_param == nr.json_param + ): + found = True + break + if found: + continue + + rules.append(nr) - rules.append(nr) def session_detect_rules_get(domain, path): - out = list() - for r in rules: - if r.domain == domain and path.startswith(r.path_prefix): - out.append(r) - return out + out = list() + for r in rules: + if r.domain == domain and path.startswith(r.path_prefix): + out.append(r) + return out + def session_create_or_update(sid, last_ids): - s = sessions.get(sid) - if s is None: - s = Session() - sessions[sid] = s - for k, v in last_ids.iteritems(): - s.last_ids[k] = v + s = sessions.get(sid) + if s is None: + s = Session() + sessions[sid] = s + for k, v in last_ids.iteritems(): + s.last_ids[k] = v + def session_update(sid, last_ids): - s = sessions.get(sid) - if s is None: - raise ValueError('unknown sid') - for k, v in last_ids.iteritems(): - s.last_ids[k] = v + s = sessions.get(sid) + if s is None: + raise ValueError("unknown sid") + for k, v in last_ids.iteritems(): + s.last_ids[k] = v + def session_get_last_ids(sid): - s = sessions.get(sid) - if s is not None: - return s.last_ids - else: - return None + s = sessions.get(sid) + if s is not None: + return s.last_ids + else: + return None + while True: - req = tnetstring.loads(sock.recv()) - method = req['method'] - args = req['args'] - print 'IN %s %s' % (method, args) - try: - resp = None - ret = None - if method == 'session-detect-rules-set': - rule_data_list = args['rules'] - rlist = list() - for rule_data in rule_data_list: - r = Rule() - r.domain = rule_data['domain'] - r.path_prefix = rule_data['path-prefix'] - r.sid_ptr = rule_data['sid-ptr'] - r.json_param = rule_data.get('json-param') - rlist.append(r) - session_detect_rules_set(rlist) - elif method == 'session-detect-rules-get': - rlist = session_detect_rules_get(args['domain'], args['path']) - ret = list() - for r in rlist: - i = {'domain': r.domain, 'path-prefix': r.path_prefix, 'sid-ptr': r.sid_ptr} - if r.json_param: - i['json-param'] = r.json_param - ret.append(i) - elif method == 'session-create-or-update': - session_create_or_update(args['sid'], args['last-ids']) - elif method == 'session-update-many': - sid_last_ids = args['sid-last-ids'] - for sid, last_ids in sid_last_ids.iteritems(): - session_update(sid, last_ids) - elif method == 'session-get-last-ids': - ret = session_get_last_ids(args['sid']) - if ret is None: - resp = {'id': req['id'], 'success': False, 'condition': 'item-not-found'} - else: - resp = {'id': req['id'], 'success': False, 'condition': 'method-not-found'} - - if resp is None: - resp = {'id': req['id'], 'success': True, 'value': ret} - except: - resp = {'id': req['id'], 'success': False, 'condition': 'general'} - - if resp['success']: - print 'OUT %s' % resp['value'] - else: - print 'OUT error, condition=%s' % resp['condition'] - sock.send(tnetstring.dumps(resp)) + req = tnetstring.loads(sock.recv()) + method = req["method"] + args = req["args"] + print("IN %s %s" % (method, args)) + try: + resp = None + ret = None + if method == "session-detect-rules-set": + rule_data_list = args["rules"] + rlist = list() + for rule_data in rule_data_list: + r = Rule() + r.domain = rule_data["domain"] + r.path_prefix = rule_data["path-prefix"] + r.sid_ptr = rule_data["sid-ptr"] + r.json_param = rule_data.get("json-param") + rlist.append(r) + session_detect_rules_set(rlist) + elif method == "session-detect-rules-get": + rlist = session_detect_rules_get(args["domain"], args["path"]) + ret = list() + for r in rlist: + i = { + "domain": r.domain, + "path-prefix": r.path_prefix, + "sid-ptr": r.sid_ptr, + } + if r.json_param: + i["json-param"] = r.json_param + ret.append(i) + elif method == "session-create-or-update": + session_create_or_update(args["sid"], args["last-ids"]) + elif method == "session-update-many": + sid_last_ids = args["sid-last-ids"] + for sid, last_ids in sid_last_ids.iteritems(): + session_update(sid, last_ids) + elif method == "session-get-last-ids": + ret = session_get_last_ids(args["sid"]) + if ret is None: + resp = { + "id": req["id"], + "success": False, + "condition": "item-not-found", + } + else: + resp = {"id": req["id"], "success": False, "condition": "method-not-found"} + + if resp is None: + resp = {"id": req["id"], "success": True, "value": ret} + except: + resp = {"id": req["id"], "success": False, "condition": "general"} + + if resp["success"]: + print("OUT %s" % resp["value"]) + else: + print("OUT error, condition=%s" % resp["condition"]) + sock.send(tnetstring.dumps(resp)) diff --git a/tools/monitorstats.py b/tools/monitorstats.py index 7115715d0..aec4ab48f 100644 --- a/tools/monitorstats.py +++ b/tools/monitorstats.py @@ -3,6 +3,7 @@ import tnetstring import zmq + def ensure_str(i): if isinstance(i, dict): out = {} @@ -15,29 +16,30 @@ def ensure_str(i): out.append(ensure_str(v)) return out elif isinstance(i, bytes): - return i.decode('utf-8') + return i.decode("utf-8") else: return i + ctx = zmq.Context() sock = ctx.socket(zmq.SUB) sock.connect(sys.argv[1]) if len(sys.argv) >= 3: - for mtype in sys.argv[2].split(','): - sock.setsockopt(zmq.SUBSCRIBE, '{} '.format(mtype).encode('utf-8')) + for mtype in sys.argv[2].split(","): + sock.setsockopt(zmq.SUBSCRIBE, "{} ".format(mtype).encode("utf-8")) else: - sock.setsockopt(zmq.SUBSCRIBE, b'') + sock.setsockopt(zmq.SUBSCRIBE, b"") while True: m_raw = sock.recv() - at = m_raw.find(b' ') + at = m_raw.find(b" ") mtype = ensure_str(m_raw[:at]) - mdata = m_raw[at + 1:] - if mdata[0] == ord(b'T'): + mdata = m_raw[at + 1 :] + if mdata[0] == ord(b"T"): m = ensure_str(tnetstring.loads(mdata[1:])) - elif mdata[0] == ord(b'J'): + elif mdata[0] == ord(b"J"): m = json.loads(mdata[1:]) else: m = mdata - print('{} {}'.format(mtype, m)) + print("{} {}".format(mtype, m)) diff --git a/tools/monitorsubs.py b/tools/monitorsubs.py index 201418024..2ba90a7e4 100644 --- a/tools/monitorsubs.py +++ b/tools/monitorsubs.py @@ -3,10 +3,12 @@ import tnetstring import zmq + class Subscription(object): - def __init__(self): - self.ttl = None - self.last_refresh = None + def __init__(self): + self.ttl = None + self.last_refresh = None + # key=(mode, channel) subs = dict() @@ -14,38 +16,38 @@ def __init__(self): ctx = zmq.Context() sock = ctx.socket(zmq.SUB) sock.connect(sys.argv[1]) -sock.setsockopt(zmq.SUBSCRIBE, 'sub ') +sock.setsockopt(zmq.SUBSCRIBE, "sub ") poller = zmq.Poller() poller.register(sock, zmq.POLLIN) while True: - socks = dict(poller.poll(1000)) - if socks.get(sock) == zmq.POLLIN: - m_raw = sock.recv() - if not m_raw.startswith('sub T'): - continue - m = tnetstring.loads(m_raw[5:]) - - now = int(time.time()) - sub_key = (m['mode'], m['channel']) - sub = subs.get(sub_key) - if m.get('unavailable'): - if sub: - del subs[sub_key] - print 'UNSUB mode=%s channel=%s' % (m['mode'], m['channel']) - else: - if not sub: - sub = Subscription() - subs[sub_key] = sub - print 'SUB mode=%s channel=%s' % (m['mode'], m['channel']) - sub.ttl = m['ttl'] - sub.last_refresh = now - - unsubs = set() - for sub_key, sub in subs.iteritems(): - if sub.last_refresh + sub.ttl <= now: - unsubs.add(sub_key) - for sub_key in unsubs: - del subs[sub_key] - print 'UNSUB mode=%s channel=%s' % (sub_key[0], sub_key[1]) + socks = dict(poller.poll(1000)) + if socks.get(sock) == zmq.POLLIN: + m_raw = sock.recv() + if not m_raw.startswith("sub T"): + continue + m = tnetstring.loads(m_raw[5:]) + + now = int(time.time()) + sub_key = (m["mode"], m["channel"]) + sub = subs.get(sub_key) + if m.get("unavailable"): + if sub: + del subs[sub_key] + print("UNSUB mode=%s channel=%s" % (m["mode"], m["channel"])) + else: + if not sub: + sub = Subscription() + subs[sub_key] = sub + print("SUB mode=%s channel=%s" % (m["mode"], m["channel"])) + sub.ttl = m["ttl"] + sub.last_refresh = now + + unsubs = set() + for sub_key, sub in subs.iteritems(): + if sub.last_refresh + sub.ttl <= now: + unsubs.add(sub_key) + for sub_key in unsubs: + del subs[sub_key] + print("UNSUB mode=%s channel=%s" % (sub_key[0], sub_key[1])) diff --git a/tools/monitorsubsock.py b/tools/monitorsubsock.py index 062ace079..9e70f9371 100644 --- a/tools/monitorsubsock.py +++ b/tools/monitorsubsock.py @@ -2,7 +2,7 @@ import zmq if len(sys.argv) < 2: - print('usage: {} [pub_spec]'.format(sys.argv[0])) + print("usage: {} [pub_spec]".format(sys.argv[0])) sys.exit(1) spec = sys.argv[1] @@ -10,15 +10,15 @@ zmq_context = zmq.Context.instance() sock = zmq_context.socket(zmq.XPUB) sock.rcvhwm = 0 -if hasattr(sock, 'immediate'): +if hasattr(sock, "immediate"): sock.immediate = 1 sock.connect(spec) while True: m = sock.recv() mtype = int(m[0]) - topic = m[1:].decode('utf-8') + topic = m[1:].decode("utf-8") if mtype == 1: - print('SUB {}'.format(topic)) + print("SUB {}".format(topic)) elif mtype == 0: - print('UNSUB {}'.format(topic)) + print("UNSUB {}".format(topic)) diff --git a/tools/mp3stream_publisher.py b/tools/mp3stream_publisher.py index 0529acac7..928fa6c36 100644 --- a/tools/mp3stream_publisher.py +++ b/tools/mp3stream_publisher.py @@ -5,17 +5,34 @@ from pubcontrol import Item from gripcontrol import GripPubControl, HttpStreamFormat -pub = GripPubControl({'control_uri': 'http://localhost:5561'}) +pub = GripPubControl({"control_uri": "http://localhost:5561"}) + def publish_worker(): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(('127.0.0.1', 5004)) - while True: - data, addr = sock.recvfrom(65536) - pub.publish('music', Item(HttpStreamFormat(data))) + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 5004)) + while True: + data, addr = sock.recvfrom(65536) + pub.publish("music", Item(HttpStreamFormat(data))) + thread = threading.Thread(target=publish_worker) thread.daemon = True thread.start() -subprocess.check_call(['gst-launch-1.0', 'filesrc', 'location=%s' % sys.argv[1], '!', 'decodebin', '!', 'queue', '!', 'lamemp3enc', '!', 'udpsink', 'clients=localhost:5004']) +subprocess.check_call( + [ + "gst-launch-1.0", + "filesrc", + "location=%s" % sys.argv[1], + "!", + "decodebin", + "!", + "queue", + "!", + "lamemp3enc", + "!", + "udpsink", + "clients=localhost:5004", + ] +) diff --git a/tools/publish.py b/tools/publish.py index ca45c686a..3a2a86653 100644 --- a/tools/publish.py +++ b/tools/publish.py @@ -22,92 +22,88 @@ import tnetstring import zmq + def ensure_utf8(i): - if isinstance(i, dict): - out = {} - for k, v in i.iteritems(): - out[ensure_utf8(k)] = ensure_utf8(v) - return out - elif isinstance(i, list): - out = [] - for v in i: - out.append(ensure_utf8(v)) - return out - elif isinstance(i, unicode): - return i.encode("utf-8") - else: - return i - -parser = argparse.ArgumentParser(description='Publish messages to Pushpin.') -parser.add_argument('channel', help='channel to send to') -parser.add_argument('content', nargs='?', default='', - help='content to use for HTTP body and WS message') -parser.add_argument('--code', type=int, - help='HTTP response code to use. default 200') -parser.add_argument('-H', '--header', action='append', - help='add HTTP response header') -parser.add_argument('--spec', default='tcp://localhost:5560', - help='zmq PUSH spec. default tcp://localhost:5560') -parser.add_argument('--close', action='store_true', - help='close streaming requests') -parser.add_argument('--id', - help='payload ID') -parser.add_argument('--prev-id', - help='payload previous ID') -parser.add_argument('--sender', - help='sender meta value') -parser.add_argument('--patch', action='store_true', - help='content is JSON patch') + if isinstance(i, dict): + out = {} + for k, v in i.iteritems(): + out[ensure_utf8(k)] = ensure_utf8(v) + return out + elif isinstance(i, list): + out = [] + for v in i: + out.append(ensure_utf8(v)) + return out + elif isinstance(i, unicode): + return i.encode("utf-8") + else: + return i + + +parser = argparse.ArgumentParser(description="Publish messages to Pushpin.") +parser.add_argument("channel", help="channel to send to") +parser.add_argument( + "content", nargs="?", default="", help="content to use for HTTP body and WS message" +) +parser.add_argument("--code", type=int, help="HTTP response code to use. default 200") +parser.add_argument("-H", "--header", action="append", help="add HTTP response header") +parser.add_argument( + "--spec", + default="tcp://localhost:5560", + help="zmq PUSH spec. default tcp://localhost:5560", +) +parser.add_argument("--close", action="store_true", help="close streaming requests") +parser.add_argument("--id", help="payload ID") +parser.add_argument("--prev-id", help="payload previous ID") +parser.add_argument("--sender", help="sender meta value") +parser.add_argument("--patch", action="store_true", help="content is JSON patch") args = parser.parse_args() headers = [] if args.header: - for h in args.header: - k, v = h.split(':', 1) - headers.append([k, v.lstrip()]) + for h in args.header: + k, v = h.split(":", 1) + headers.append([k, v.lstrip()]) meta = dict() formats = dict() if args.content: - hr = {} - if args.patch: - hr['body-patch'] = ensure_utf8(json.loads(args.content)) - else: - hr['body'] = args.content + '\n' - if args.code is not None: - hr['code'] = args.code - if headers: - hr['headers'] = headers - formats['http-response'] = hr + hr = {} + if args.patch: + hr["body-patch"] = ensure_utf8(json.loads(args.content)) + else: + hr["body"] = args.content + "\n" + if args.code is not None: + hr["code"] = args.code + if headers: + hr["headers"] = headers + formats["http-response"] = hr if args.close: - formats['http-stream'] = {'action': 'close'} + formats["http-stream"] = {"action": "close"} elif args.content and not args.patch: - formats['http-stream'] = {'content': args.content + '\n'} + formats["http-stream"] = {"content": args.content + "\n"} if args.content and not args.patch: - formats['ws-message'] = {'content': args.content} + formats["ws-message"] = {"content": args.content} if not formats: - print 'error: nothing to send' - sys.exit(1) + print("error: nothing to send") + sys.exit(1) if args.sender: - meta['sender'] = args.sender + meta["sender"] = args.sender -item = { - 'channel': args.channel, - 'formats': formats -} +item = {"channel": args.channel, "formats": formats} if args.id: - item['id'] = args.id + item["id"] = args.id if args.prev_id: - item['prev-id'] = args.prev_id + item["prev-id"] = args.prev_id if meta: - item['meta'] = meta + item["meta"] = meta ctx = zmq.Context() sock = ctx.socket(zmq.PUSH) @@ -115,4 +111,4 @@ def ensure_utf8(i): sock.send(tnetstring.dumps(item)) -print 'Published' +print("Published") diff --git a/tools/publish2.py b/tools/publish2.py index 1133ab30a..fe6744b08 100644 --- a/tools/publish2.py +++ b/tools/publish2.py @@ -6,7 +6,7 @@ import zmq if len(sys.argv) < 3: - print('usage: {} [channel] [content]'.format(sys.argv[0])) + print("usage: {} [channel] [content]".format(sys.argv[0])) sys.exit(1) channel = sys.argv[1] @@ -29,7 +29,7 @@ # a PULL socket for input. sock = ctx.socket(zmq.XPUB) -sock.connect('tcp://localhost:5562') +sock.connect("tcp://localhost:5562") poller = zmq.Poller() poller.register(sock, zmq.POLLIN) @@ -42,22 +42,18 @@ socks = dict(poller.poll(500 - elapsed)) if socks.get(sock) == zmq.POLLIN: m = sock.recv() - if m[0] == 1 and m[1:].decode('utf-8') == channel: + if m[0] == 1 and m[1:].decode("utf-8") == channel: # subscription ready break -content = content.encode('utf-8') +content = content.encode("utf-8") -hr = {b'body': content + b'\n'} -hs = {b'content': content + b'\n'} -ws = {b'content': content} -formats = { - b'http-response': hr, - b'http-stream': hs, - b'ws-message': ws -} -item = {b'formats': formats} +hr = {b"body": content + b"\n"} +hs = {b"content": content + b"\n"} +ws = {b"content": content} +formats = {b"http-response": hr, b"http-stream": hs, b"ws-message": ws} +item = {b"formats": formats} -sock.send_multipart([channel.encode('utf-8'), tnetstring.dumps(item)]) +sock.send_multipart([channel.encode("utf-8"), tnetstring.dumps(item)]) -print('Published') +print("Published") diff --git a/tools/recover.py b/tools/recover.py index 13886a24c..6705d5175 100644 --- a/tools/recover.py +++ b/tools/recover.py @@ -7,9 +7,9 @@ sock = zmq.Context.instance().socket(zmq.REQ) sock.connect(command_uri) -req = {'method': 'recover'} +req = {"method": "recover"} sock.send(tnetstring.dumps(req)) resp = tnetstring.loads(sock.recv()) -if not resp.get('success'): - raise ValueError('request failed: %s' % resp) +if not resp.get("success"): + raise ValueError("request failed: %s" % resp) diff --git a/tools/sourcebroker.py b/tools/sourcebroker.py index f125d63b6..f863a61ea 100644 --- a/tools/sourcebroker.py +++ b/tools/sourcebroker.py @@ -3,8 +3,8 @@ import zmq if len(sys.argv) < 3: - print 'usage: %s [pub_spec] [pull_spec]' % sys.argv[0] - sys.exit(1) + print("usage: %s [pub_spec] [pull_spec]" % sys.argv[0]) + sys.exit(1) pub_spec = sys.argv[1] pull_spec = sys.argv[2] @@ -24,22 +24,22 @@ subs = set() while True: - socks = dict(poller.poll()) - if socks.get(pull_sock) == zmq.POLLIN: - m = tnetstring.loads(pull_sock.recv()) - channel = m['channel'] - if channel in subs: - del m['channel'] - pub_sock.send_multipart([channel, tnetstring.dumps(m)]) - elif socks.get(pub_sock) == zmq.POLLIN: - m = pub_sock.recv() - mtype = m[0] - topic = m[1:] - if mtype == '\x01': - assert(topic not in subs) - print 'subscribing [%s]' % topic - subs.add(topic) - elif mtype == '\x00': - assert(topic in subs) - print 'unsubscribing [%s]' % topic - subs.remove(topic) + socks = dict(poller.poll()) + if socks.get(pull_sock) == zmq.POLLIN: + m = tnetstring.loads(pull_sock.recv()) + channel = m["channel"] + if channel in subs: + del m["channel"] + pub_sock.send_multipart([channel, tnetstring.dumps(m)]) + elif socks.get(pub_sock) == zmq.POLLIN: + m = pub_sock.recv() + mtype = m[0] + topic = m[1:] + if mtype == "\x01": + assert topic not in subs + print("subscribing [%s]" % topic) + subs.add(topic) + elif mtype == "\x00": + assert topic in subs + print("unsubscribing [%s]" % topic) + subs.remove(topic) diff --git a/tools/zhttp/basichandler.py b/tools/zhttp/basichandler.py index b363d1513..e81a7ba3d 100644 --- a/tools/zhttp/basichandler.py +++ b/tools/zhttp/basichandler.py @@ -5,13 +5,13 @@ import tnetstring import zmq -instance_id = 'basichandler.{}'.format(os.getpid()).encode() +instance_id = "basichandler.{}".format(os.getpid()).encode() ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc://client-out') +in_sock.connect("ipc://client-out") out_sock = ctx.socket(zmq.PUB) -out_sock.connect('ipc://client-in') +out_sock.connect("ipc://client-in") # await subscription time.sleep(0.01) @@ -19,15 +19,15 @@ while True: m_raw = in_sock.recv() req = tnetstring.loads(m_raw[1:]) - print('IN {}'.format(req)) + print("IN {}".format(req)) resp = {} - resp[b'from'] = instance_id - resp[b'id'] = req[b'id'] - resp[b'code'] = 200 - resp[b'reason'] = b'OK' - resp[b'headers'] = [[b'Content-Type', b'text/plain']] - resp[b'body'] = b'hello world\n' + resp[b"from"] = instance_id + resp[b"id"] = req[b"id"] + resp[b"code"] = 200 + resp[b"reason"] = b"OK" + resp[b"headers"] = [[b"Content-Type", b"text/plain"]] + resp[b"body"] = b"hello world\n" - print('OUT {}'.format(resp)) - out_sock.send(req[b'from'] + b' T' + tnetstring.dumps(resp)) + print("OUT {}".format(resp)) + out_sock.send(req[b"from"] + b" T" + tnetstring.dumps(resp)) diff --git a/tools/zhttp/get.py b/tools/zhttp/get.py index e36411598..b5ea0ac6c 100644 --- a/tools/zhttp/get.py +++ b/tools/zhttp/get.py @@ -4,32 +4,32 @@ import zmq if len(sys.argv) < 2: - print('usage: {} [url]'.format(sys.argv[0])) + print("usage: {} [url]".format(sys.argv[0])) sys.exit(1) ctx = zmq.Context() sock = ctx.socket(zmq.REQ) -sock.connect('ipc://server') +sock.connect("ipc://server") req = { - b'method': b'GET', - b'uri': sys.argv[1].encode('utf-8'), - #b'follow-redirects': True, - #b'ignore-tls-errors': True, + b"method": b"GET", + b"uri": sys.argv[1].encode("utf-8"), + # b'follow-redirects': True, + # b'ignore-tls-errors': True, } -sock.send(b'T' + tnetstring.dumps(req)) +sock.send(b"T" + tnetstring.dumps(req)) resp = tnetstring.loads(sock.recv()[1:]) -if b'type' in resp and resp[b'type'] == b'error': - print('error: {}'.format(resp[b'condition'])) +if b"type" in resp and resp[b"type"] == b"error": + print("error: {}".format(resp[b"condition"])) sys.exit(1) -print('code={} reason=[{}]'.format(resp[b'code'], resp[b'reason'])) -for h in resp[b'headers']: - print('{}: {}'.format(h[0], h[1])) +print("code={} reason=[{}]".format(resp[b"code"], resp[b"reason"])) +for h in resp[b"headers"]: + print("{}: {}".format(h[0], h[1])) -if b'body' in resp: - print('\n{}'.format(resp[b'body'])) +if b"body" in resp: + print("\n{}".format(resp[b"body"])) else: - print('\n') + print("\n") diff --git a/tools/zhttp/getstream.py b/tools/zhttp/getstream.py index 62d4f6f1c..a2db27a54 100644 --- a/tools/zhttp/getstream.py +++ b/tools/zhttp/getstream.py @@ -4,54 +4,62 @@ import tnetstring import zmq -client_id = b'getstream.py' +client_id = b"getstream.py" ctx = zmq.Context() out_sock = ctx.socket(zmq.PUSH) -out_sock.connect('ipc://server-in') +out_sock.connect("ipc://server-in") out_stream_sock = ctx.socket(zmq.ROUTER) -out_stream_sock.connect('ipc://server-in-stream') +out_stream_sock.connect("ipc://server-in-stream") in_sock = ctx.socket(zmq.SUB) in_sock.setsockopt(zmq.SUBSCRIBE, client_id) -in_sock.connect('ipc://server-out') +in_sock.connect("ipc://server-out") time.sleep(0.5) -rid = str(uuid.uuid4()).encode('utf-8') +rid = str(uuid.uuid4()).encode("utf-8") inseq = 0 outseq = 0 -out_sock.send(b'T' + tnetstring.dumps({ - b'from': client_id, - b'id': rid, - b'seq': outseq, - b'method': b'GET', - b'uri': sys.argv[1].encode('utf-8'), - b'stream': True, - b'credits': 8192, -})) +out_sock.send( + b"T" + + tnetstring.dumps( + { + b"from": client_id, + b"id": rid, + b"seq": outseq, + b"method": b"GET", + b"uri": sys.argv[1].encode("utf-8"), + b"stream": True, + b"credits": 8192, + } + ) +) outseq += 1 while True: buf = in_sock.recv() - at = buf.find(b' ') + at = buf.find(b" ") receiver = buf[:at] - indata = tnetstring.loads(buf[at + 2:]) - if indata[b'id'] != rid: + indata = tnetstring.loads(buf[at + 2 :]) + if indata[b"id"] != rid: continue - print('IN: {}'.format(indata)) - assert(indata[b'seq'] == inseq) + print("IN: {}".format(indata)) + assert indata[b"seq"] == inseq inseq += 1 - if (b'type' in indata and (indata[b'type'] == b'error' or indata[b'type'] == b'cancel')) or (b'type' not in indata and b'more' not in indata): + if ( + b"type" in indata + and (indata[b"type"] == b"error" or indata[b"type"] == b"cancel") + ) or (b"type" not in indata and b"more" not in indata): break - raddr = indata[b'from'] - if b'body' in indata and len(indata[b'body']) > 0: + raddr = indata[b"from"] + if b"body" in indata and len(indata[b"body"]) > 0: outdata = { - b'id': rid, - b'from': client_id, - b'seq': outseq, - b'type': b'credit', - b'credits': len(indata[b'body']), + b"id": rid, + b"from": client_id, + b"seq": outseq, + b"type": b"credit", + b"credits": len(indata[b"body"]), } - print('OUT: {}'.format(outdata)) - out_stream_sock.send_multipart([raddr, b'', b'T' + tnetstring.dumps(outdata)]) + print("OUT: {}".format(outdata)) + out_stream_sock.send_multipart([raddr, b"", b"T" + tnetstring.dumps(outdata)]) outseq += 1 diff --git a/tools/zhttp/holdhandler.py b/tools/zhttp/holdhandler.py index 498a4a09c..752d39ad0 100644 --- a/tools/zhttp/holdhandler.py +++ b/tools/zhttp/holdhandler.py @@ -10,21 +10,22 @@ CONN_TTL = 60000 EXPIRE_INTERVAL = 60000 -instance_id = 'holdhandler.{}'.format(os.getpid()).encode('utf-8') +instance_id = "holdhandler.{}".format(os.getpid()).encode("utf-8") ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc://client-out') +in_sock.connect("ipc://client-out") in_stream_sock = ctx.socket(zmq.ROUTER) in_stream_sock.identity = instance_id -in_stream_sock.connect('ipc://client-out-stream') +in_stream_sock.connect("ipc://client-out-stream") out_sock = ctx.socket(zmq.PUB) -out_sock.connect('ipc://client-in') +out_sock.connect("ipc://client-in") poller = zmq.Poller() poller.register(in_sock, zmq.POLLIN) poller.register(in_stream_sock, zmq.POLLIN) + class Connection(object): def __init__(self, rid): self.rid = rid @@ -32,44 +33,46 @@ def __init__(self, rid): self.exp_time = None def send_msg(self, msg): - msg[b'from'] = instance_id - msg[b'id'] = self.rid[1] - msg[b'seq'] = self.seq + msg[b"from"] = instance_id + msg[b"id"] = self.rid[1] + msg[b"seq"] = self.seq self.seq += 1 - print('OUT {} {}'.format(self.rid[0], msg)) - out_sock.send(self.rid[0] + b' T' + tnetstring.dumps(msg)) + print("OUT {} {}".format(self.rid[0], msg)) + out_sock.send(self.rid[0] + b" T" + tnetstring.dumps(msg)) def send_header(self): msg = {} - msg[b'code'] = 200 - msg[b'reason'] = b'OK' - msg[b'headers'] = [[b'Content-Type', b'text/plain']] - msg[b'more'] = True + msg[b"code"] = 200 + msg[b"reason"] = b"OK" + msg[b"headers"] = [[b"Content-Type", b"text/plain"]] + msg[b"more"] = True self.send_msg(msg) def send_body(self, data): msg = {} - msg[b'body'] = data - msg[b'more'] = True + msg[b"body"] = data + msg[b"more"] = True self.send_msg(msg) + def send_body(to_addr, conns, data): ids = [] for c in conns: - ids.append({b'id': c.rid[1], b'seq': c.seq}) + ids.append({b"id": c.rid[1], b"seq": c.seq}) c.seq += 1 msg = {} - msg[b'from'] = instance_id - msg[b'id'] = ids - msg[b'body'] = data - msg[b'more'] = True + msg[b"from"] = instance_id + msg[b"id"] = ids + msg[b"body"] = data + msg[b"more"] = True + + print("OUT {} {}".format(to_addr, msg)) + out_sock.send(to_addr + b" T" + tnetstring.dumps(msg)) - print('OUT {} {}'.format(to_addr, msg)) - out_sock.send(to_addr + b' T' + tnetstring.dumps(msg)) conns = {} last_exp_time = int(time.time()) @@ -89,16 +92,16 @@ def send_body(to_addr, conns, data): if m_raw is not None: req = tnetstring.loads(m_raw[1:]) - print('IN {}'.format(req)) + print("IN {}".format(req)) - m_from = req[b'from'] - m_id = req[b'id'] - m_type = req.get(b'type', b'') + m_from = req[b"from"] + m_id = req[b"id"] + m_type = req.get(b"type", b"") ids = [] if isinstance(m_id, list): for id_seq in m_id: - ids.append(id_seq[b'id']) + ids.append(id_seq[b"id"]) else: ids.append(m_id) @@ -122,26 +125,30 @@ def send_body(to_addr, conns, data): c.exp_time = now + CONN_TTL c.send_header() elif c: - if m_type == b'keep-alive': + if m_type == b"keep-alive": dt = datetime.datetime.utcnow() ts = calendar.timegm(dt.timetuple()) body = ( - 'id: TCPKaliMsgTS-{:016x}.\n' - 'event: message\n' - 'data: {:04}-{:02}-{:02}T{:02}:{:02}:{:02}\n\n' - ).format( - (ts * 1000000) + dt.microsecond, - dt.year, - dt.month, - dt.day, - dt.hour, - dt.minute, - dt.second - ).encode() + ( + "id: TCPKaliMsgTS-{:016x}.\n" + "event: message\n" + "data: {:04}-{:02}-{:02}T{:02}:{:02}:{:02}\n\n" + ) + .format( + (ts * 1000000) + dt.microsecond, + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + ) + .encode() + ) send_body(m_from, known_conns, body) - elif m_type == b'cancel': + elif m_type == b"cancel": for c in known_conns: del conns[c.rid] @@ -153,5 +160,5 @@ def send_body(to_addr, conns, data): if now >= c.exp_time: to_remove.append(rid) for rid in to_remove: - print('expired {}'.format(rid)) + print("expired {}".format(rid)) del conns[rid] diff --git a/tools/zhttp/printreq.py b/tools/zhttp/printreq.py index a86bbb7cb..d0ecdc38e 100644 --- a/tools/zhttp/printreq.py +++ b/tools/zhttp/printreq.py @@ -5,9 +5,9 @@ ctx = zmq.Context() sock = ctx.socket(zmq.PULL) -sock.connect('ipc://client-out') +sock.connect("ipc://client-out") while True: m = sock.recv_multipart() req = tnetstring.loads(m[0][1:]) - print('{} {}'.format(req[b'from'].decode(), req[b'id'].decode())) + print("{} {}".format(req[b"from"].decode(), req[b"id"].decode())) diff --git a/tools/zhttp/reqhandler.py b/tools/zhttp/reqhandler.py index 311231f13..5035381a0 100644 --- a/tools/zhttp/reqhandler.py +++ b/tools/zhttp/reqhandler.py @@ -5,19 +5,19 @@ ctx = zmq.Context() sock = ctx.socket(zmq.REP) -sock.connect('ipc://client') +sock.connect("ipc://client") while True: m_raw = sock.recv() req = tnetstring.loads(m_raw[1:]) - print('IN {}'.format(req)) + print("IN {}".format(req)) resp = {} - resp[b'id'] = req[b'id'] - resp[b'code'] = 200 - resp[b'reason'] = b'OK' - resp[b'headers'] = [[b'Content-Type', b'text/plain']] - resp[b'body'] = b'hello world\n' + resp[b"id"] = req[b"id"] + resp[b"code"] = 200 + resp[b"reason"] = b"OK" + resp[b"headers"] = [[b"Content-Type", b"text/plain"]] + resp[b"body"] = b"hello world\n" - print('OUT {}'.format(resp)) - sock.send(b'T' + tnetstring.dumps(resp)) + print("OUT {}".format(resp)) + sock.send(b"T" + tnetstring.dumps(resp)) diff --git a/tools/zhttp/sendresp.py b/tools/zhttp/sendresp.py index 7827b8b34..77eefa250 100644 --- a/tools/zhttp/sendresp.py +++ b/tools/zhttp/sendresp.py @@ -11,19 +11,19 @@ ctx = zmq.Context() sock = ctx.socket(zmq.PUB) -sock.connect('ipc://client-in') +sock.connect("ipc://client-in") # await subscription time.sleep(0.01) resp = {} -resp[b'from'] = b'sendresp' -resp[b'id'] = rid -resp[b'code'] = 200 -resp[b'reason'] = b'OK' -resp[b'headers'] = [[b'Content-Type', b'text/plain']] -resp[b'body'] = '{}\n'.format(body).encode() +resp[b"from"] = b"sendresp" +resp[b"id"] = rid +resp[b"code"] = 200 +resp[b"reason"] = b"OK" +resp[b"headers"] = [[b"Content-Type", b"text/plain"]] +resp[b"body"] = "{}\n".format(body).encode() -m = [addr + b' T' + tnetstring.dumps(resp)] +m = [addr + b" T" + tnetstring.dumps(resp)] sock.send_multipart(m) diff --git a/tools/zhttp/streamhandler.py b/tools/zhttp/streamhandler.py index c5a278fb3..11b562cdb 100644 --- a/tools/zhttp/streamhandler.py +++ b/tools/zhttp/streamhandler.py @@ -4,16 +4,16 @@ import tnetstring import zmq -instance_id = 'streamhandler.{}'.format(os.getpid()).encode('utf-8') +instance_id = "streamhandler.{}".format(os.getpid()).encode("utf-8") ctx = zmq.Context() in_sock = ctx.socket(zmq.PULL) -in_sock.connect('ipc://client-out') +in_sock.connect("ipc://client-out") in_stream_sock = ctx.socket(zmq.ROUTER) in_stream_sock.identity = instance_id -in_stream_sock.connect('ipc://client-out-stream') +in_stream_sock.connect("ipc://client-out-stream") out_sock = ctx.socket(zmq.PUB) -out_sock.connect('ipc://client-in') +out_sock.connect("ipc://client-in") poller = zmq.Poller() poller.register(in_sock, zmq.POLLIN) @@ -31,60 +31,60 @@ continue req = tnetstring.loads(m_raw[1:]) - print('IN {}'.format(req)) + print("IN {}".format(req)) - if req.get(b'type'): + if req.get(b"type"): # skip all non-data messages continue - if req.get(b'uri', b'').startswith(b'ws'): + if req.get(b"uri", b"").startswith(b"ws"): resp = {} - resp[b'from'] = instance_id - resp[b'id'] = req[b'id'] - resp[b'seq'] = 0 - resp[b'code'] = 101 - resp[b'reason'] = b'Switching Protocols' - resp[b'credits'] = 1024 + resp[b"from"] = instance_id + resp[b"id"] = req[b"id"] + resp[b"seq"] = 0 + resp[b"code"] = 101 + resp[b"reason"] = b"Switching Protocols" + resp[b"credits"] = 1024 - print('OUT {} {}'.format(req[b'from'], resp)) - out_sock.send(req[b'from'] + b' T' + tnetstring.dumps(resp)) + print("OUT {} {}".format(req[b"from"], resp)) + out_sock.send(req[b"from"] + b" T" + tnetstring.dumps(resp)) resp = {} - resp[b'from'] = instance_id - resp[b'id'] = req[b'id'] - resp[b'seq'] = 1 - resp[b'body'] = b'hello world' + resp[b"from"] = instance_id + resp[b"id"] = req[b"id"] + resp[b"seq"] = 1 + resp[b"body"] = b"hello world" - print('OUT {} {}'.format(req[b'from'], resp)) - out_sock.send(req[b'from'] + b' T' + tnetstring.dumps(resp)) + print("OUT {} {}".format(req[b"from"], resp)) + out_sock.send(req[b"from"] + b" T" + tnetstring.dumps(resp)) resp = {} - resp[b'from'] = instance_id - resp[b'id'] = req[b'id'] - resp[b'seq'] = 2 - resp[b'type'] = b'close' + resp[b"from"] = instance_id + resp[b"id"] = req[b"id"] + resp[b"seq"] = 2 + resp[b"type"] = b"close" - print('OUT {} {}'.format(req[b'from'], resp)) - out_sock.send(req[b'from'] + b' T' + tnetstring.dumps(resp)) + print("OUT {} {}".format(req[b"from"], resp)) + out_sock.send(req[b"from"] + b" T" + tnetstring.dumps(resp)) else: resp = {} - resp[b'from'] = instance_id - resp[b'id'] = req[b'id'] - resp[b'seq'] = 0 - resp[b'code'] = 200 - resp[b'reason'] = b'OK' - resp[b'headers'] = [[b'Content-Type', b'text/plain']] - resp[b'more'] = True - resp[b'credits'] = 1024 - - print('OUT {} {}'.format(req[b'from'], resp)) - out_sock.send(req[b'from'] + b' T' + tnetstring.dumps(resp)) + resp[b"from"] = instance_id + resp[b"id"] = req[b"id"] + resp[b"seq"] = 0 + resp[b"code"] = 200 + resp[b"reason"] = b"OK" + resp[b"headers"] = [[b"Content-Type", b"text/plain"]] + resp[b"more"] = True + resp[b"credits"] = 1024 + + print("OUT {} {}".format(req[b"from"], resp)) + out_sock.send(req[b"from"] + b" T" + tnetstring.dumps(resp)) resp = {} - resp[b'from'] = instance_id - resp[b'id'] = req[b'id'] - resp[b'seq'] = 1 - resp[b'body'] = b'hello world\n' + resp[b"from"] = instance_id + resp[b"id"] = req[b"id"] + resp[b"seq"] = 1 + resp[b"body"] = b"hello world\n" - print('OUT {} {}'.format(req[b'from'], resp)) - out_sock.send(req[b'from'] + b' T' + tnetstring.dumps(resp)) + print("OUT {} {}".format(req[b"from"], resp)) + out_sock.send(req[b"from"] + b" T" + tnetstring.dumps(resp)) diff --git a/tools/zhttpreqhandler.py b/tools/zhttpreqhandler.py index 21ed16b79..cacaf7792 100644 --- a/tools/zhttpreqhandler.py +++ b/tools/zhttpreqhandler.py @@ -5,19 +5,17 @@ zmq_context = zmq.Context() sock = zmq_context.socket(zmq.REP) -sock.connect('ipc:///tmp/zhttpreqhandler') +sock.connect("ipc:///tmp/zhttpreqhandler") while True: - req = tnetstring.loads(sock.recv()[1:]) + req = tnetstring.loads(sock.recv()[1:]) - resp = { - 'id': req['id'], - 'code': 200, - 'reason': 'OK', - 'headers': [ - ['Content-Type', 'text/plain'] - ], - 'body': 'hello there\n' - } + resp = { + "id": req["id"], + "code": 200, + "reason": "OK", + "headers": [["Content-Type", "text/plain"]], + "body": "hello there\n", + } - sock.send('T' + tnetstring.dumps(resp)) + sock.send("T" + tnetstring.dumps(resp))