Skip to content

Commit 8850e8c

Browse files
sbylica-splunksbylica-splunk
authored
feat: configurator tool script (#2834)
* feat: configurator tool * feat: added docs for configuration helper
1 parent 274740e commit 8850e8c

File tree

3 files changed

+366
-0
lines changed

3 files changed

+366
-0
lines changed

‎configuration-tool.sh‎

Lines changed: 343 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,343 @@
1+
#!/bin/bash
2+
# filepath: configuration-tool.sh
3+
4+
# SC4S Configuration Tool
5+
# Generates a customized env_file based on user requirements
6+
7+
set -e
8+
9+
# Colors for better UX
10+
RED='\033[0;31m'
11+
GREEN='\033[0;32m'
12+
YELLOW='\033[1;33m'
13+
BLUE='\033[0;34m'
14+
NC='\033[0m' # No Color
15+
16+
# Initialize variables
17+
OUTPUT_FILE="env_file"
18+
SPLUNK_URL=""
19+
HEC_TOKEN=""
20+
TLS_VERIFY="yes"
21+
EXPECTED_EPS=1000
22+
PROTOCOL="both"
23+
24+
SC4S_SOURCE_LISTEN_UDP_SOCKETS=2
25+
SC4S_SOURCE_UDP_FETCH_LIMIT=1000
26+
SC4S_ENABLE_EBPF="no"
27+
SC4S_EBPF_NO_SOCKETS=4
28+
29+
PARALLELIZE="no"
30+
SC4S_PARALLELIZE_NO_PARTITION=4
31+
SC4S_SOURCE_TCP_IW_USE="no"
32+
SC4S_SOURCE_TCP_IW_SIZE=1000000
33+
34+
SC4S_SOURCE_UDP_SO_RCVBUFF=-1
35+
SC4S_SOURCE_TCP_SO_RCVBUFF=-1
36+
37+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE="yes"
38+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE="no"
39+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE=10241024
40+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH=15000
41+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE=53687091200
42+
43+
echo ""
44+
echo "${BLUE}================================================${NC}"
45+
echo "${BLUE} SC4S Configuration Tool${NC}"
46+
echo "${BLUE}================================================${NC}"
47+
echo ""
48+
echo "This tool will help you generate an optimized SC4S configuration"
49+
echo "based on your requirements."
50+
echo ""
51+
52+
# Function to ask yes/no questions
53+
ask_yes_no() {
54+
local question="$1"
55+
local default="$2"
56+
local response
57+
58+
while true; do
59+
if [[ "$default" == "yes" ]]; then
60+
read -p "$question [Y/n]: " response
61+
response=${response:-y}
62+
else
63+
read -p "$question [y/N]: " response
64+
response=${response:-n}
65+
fi
66+
67+
case "$response" in
68+
[Yy]*|yes*|Yes*|YES* ) echo "yes"; break;;
69+
[Nn]*|no*|No*|NO* ) echo "no"; break;;
70+
* ) echo "Please answer yes or no.";;
71+
esac
72+
done
73+
}
74+
75+
echo "${GREEN}=== Splunk Configuration ===${NC}"
76+
77+
# Splunk HEC URL
78+
read -p "Enter your Splunk HEC URL (e.g., https://your.splunk.instance:8088): " SPLUNK_URL
79+
80+
# HEC Token
81+
read -p "Enter your Splunk HEC Token: " HEC_TOKEN
82+
83+
# TLS Verification
84+
TLS_VERIFY=$(ask_yes_no "Verify SSL/TLS certificates?" "yes")
85+
86+
echo ""
87+
echo "${GREEN}=== Performance Configuration ===${NC}"
88+
89+
# Protocol selection
90+
echo ""
91+
echo "Protocol optimisation:"
92+
echo "1) UDP only (faster, may lose messages)"
93+
echo "2) TCP only (reliable, slower)"
94+
echo "3) Both UDP and TCP (default)"
95+
read -p "Select protocol [3]: " protocol_choice
96+
protocol_choice=${protocol_choice:-3}
97+
case "$protocol_choice" in
98+
1) PROTOCOL="udp";;
99+
2) PROTOCOL="tcp";;
100+
3) PROTOCOL="both";;
101+
*) PROTOCOL="both";;
102+
esac
103+
104+
# # Expected EPS
105+
# read -p "Expected events per second (EPS) [1000]: " input_eps
106+
# EXPECTED_EPS=${input_eps:-1000}
107+
108+
# # Calculate optimal settings based on EPS
109+
# if [ "$EXPECTED_EPS" -gt 10000 ]; then
110+
# UDP_SOCKETS=4
111+
# PARALLELIZE="yes"
112+
# echo -e "${YELLOW}High EPS detected (${EXPECTED_EPS}), enabling performance optimizations${NC}"
113+
# elif [ "$EXPECTED_EPS" -gt 5000 ]; then
114+
# UDP_SOCKETS=3
115+
# PARALLELIZE="yes"
116+
# echo -e "${YELLOW}Medium-high EPS detected (${EXPECTED_EPS}), using moderate optimizations${NC}"
117+
# elif [ "$EXPECTED_EPS" -gt 1000 ]; then
118+
# UDP_SOCKETS=2
119+
# PARALLELIZE="no"
120+
# fi
121+
122+
# Advanced UDP options
123+
echo ""
124+
echo "${GREEN}=== Advanced UDP Options ===${NC}"
125+
126+
# UDP fetch limit overrides
127+
if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
128+
ADJUST_FETCH_LIMIT=$(ask_yes_no "Adjust fetch limit for UDP" "no")
129+
130+
if [[ "$ADJUST_FETCH_LIMIT" == "yes" ]]; then
131+
read -p "UDP fetch limit [$SC4S_SOURCE_UDP_FETCH_LIMIT]: " input_udp_fetch_limit
132+
SC4S_SOURCE_UDP_FETCH_LIMIT=${input_udp_fetch_limit:-$SC4S_SOURCE_UDP_FETCH_LIMIT}
133+
fi
134+
fi
135+
136+
# UDP listen socket overrides
137+
if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
138+
ADJUST_LISTEN_SOCKETS=$(ask_yes_no "Adjust number of UDP listen sockets?" "no")
139+
140+
if [[ "$ADJUST_LISTEN_SOCKETS" == "yes" ]]; then
141+
read -p "UDP listen sockets [$SC4S_SOURCE_LISTEN_UDP_SOCKETS]: " input_udp_sockets
142+
SC4S_SOURCE_LISTEN_UDP_SOCKETS=${input_udp_sockets:-$SC4S_SOURCE_LISTEN_UDP_SOCKETS}
143+
fi
144+
fi
145+
146+
# UDP receiving buffer overrides
147+
if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
148+
read -p "Tune UDP receiving buffer (-1 to skip, default 17039360 bytes) [$SC4S_SOURCE_UDP_SO_RCVBUFF]: " input_udp_rcvbuff
149+
SC4S_SOURCE_UDP_SO_RCVBUFF=${input_udp_rcvbuff:-$SC4S_SOURCE_UDP_SO_RCVBUFF}
150+
fi
151+
152+
# UDP eBPF options
153+
if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
154+
SC4S_ENABLE_EBPF=$(ask_yes_no "Enable eBPF?" "$SC4S_ENABLE_EBPF")
155+
if [[ "$SC4S_ENABLE_EBPF" == "yes" ]]; then
156+
read -p "Number of eBPF sockets [4]: " input_ebpf_sockets
157+
SC4S_EBPF_NO_SOCKETS=${input_ebpf_sockets:-4}
158+
fi
159+
fi
160+
161+
# Advanced TCP options
162+
echo ""
163+
echo "${GREEN}=== Advanced TCP Options ===${NC}"
164+
165+
# TCP receiving buffer overrides
166+
if [[ "$PROTOCOL" == "tcp" || "$PROTOCOL" == "both" ]]; then
167+
read -p "Tune TCP receiving buffer (-1 to skip, default 17039360 bytes) [$SC4S_SOURCE_TCP_SO_RCVBUFF]: " input_tcp_rcvbuff
168+
SC4S_SOURCE_TCP_SO_RCVBUFF=${input_tcp_rcvbuff:-$SC4S_SOURCE_TCP_SO_RCVBUFF}
169+
fi
170+
171+
# TCP parallelization
172+
if [[ "$PROTOCOL" == "tcp" || "$PROTOCOL" == "both" ]]; then
173+
PARALLELIZE=$(ask_yes_no "Enable TCP parallelization?" "$PARALLELIZE")
174+
if [[ "$PARALLELIZE" == "yes" ]]; then
175+
read -p "Number of partitions for parallelization [4]: " input_partitions
176+
SC4S_PARALLELIZE_NO_PARTITION=${input_partitions:-4}
177+
fi
178+
fi
179+
180+
# TCP IW settings
181+
if [[ "$PROTOCOL" == "tcp" || "$PROTOCOL" == "both" ]]; then
182+
SC4S_SOURCE_TCP_IW_USE=$(ask_yes_no "Tune static window size?" "$SC4S_SOURCE_TCP_IW_USE")
183+
if [[ "$SC4S_SOURCE_TCP_IW_USE" == "yes" ]]; then
184+
read -p "Input window size [1000000]: " input_iw_size
185+
SC4S_SOURCE_TCP_IW_SIZE=${input_iw_size:-1000000}
186+
fi
187+
fi
188+
189+
# Disk Buffer Configuration
190+
echo ""
191+
echo "${GREEN}=== Disk Buffer Configuration ===${NC}"
192+
193+
ADJUST_DISKBUFF=$(ask_yes_no "Adjust disk buffer settings?" "no")
194+
195+
if [[ "$ADJUST_DISKBUFF" == "yes" ]]; then
196+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE=$(ask_yes_no "Enable local disk buffering?" "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE")
197+
198+
if [[ "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE" == "yes" ]]; then
199+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE=$(ask_yes_no "Enable reliable disk buffering (recommended: no for normal buffering)?" "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE")
200+
201+
if [[ "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE" == "yes" ]]; then
202+
read -p "Worker memory buffer size in bytes (for reliable buffering) [$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE]: " input_membufsize
203+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE=${input_membufsize:-$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE}
204+
else
205+
read -p "Worker memory buffer size in message count (for normal buffering) [$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH]: " input_membuflength
206+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH=${input_membuflength:-$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH}
207+
fi
208+
209+
read -p "Disk buffer size in bytes (default 50GB per worker) [$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE]: " input_diskbufsize
210+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE=${input_diskbufsize:-$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE}
211+
fi
212+
fi
213+
214+
# Output file
215+
echo ""
216+
read -p "Output filename [$OUTPUT_FILE]: " input_output
217+
OUTPUT_FILE=${input_output:-$OUTPUT_FILE}
218+
219+
# Generate configuration
220+
echo ""
221+
echo "${BLUE}Generating configuration...${NC}"
222+
223+
cat > "$OUTPUT_FILE" << EOF
224+
# SC4S Configuration - Generated by configuration tool
225+
# Expected EPS: $EXPECTED_EPS
226+
# Protocol: $PROTOCOL
227+
# Generated on: $(date)
228+
229+
# === Splunk HEC Configuration ===
230+
SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=$SPLUNK_URL
231+
SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=$HEC_TOKEN
232+
EOF
233+
234+
if [ "$TLS_VERIFY" == "no" ]; then
235+
cat >> "$OUTPUT_FILE" << EOF
236+
SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no
237+
EOF
238+
fi
239+
240+
cat >> "$OUTPUT_FILE" << EOF
241+
242+
# === Performance Configuration ===
243+
EOF
244+
245+
if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
246+
cat >> "$OUTPUT_FILE" << EOF
247+
SC4S_SOURCE_UDP_FETCH_LIMIT=$SC4S_SOURCE_UDP_FETCH_LIMIT
248+
SC4S_SOURCE_LISTEN_UDP_SOCKETS=$SC4S_SOURCE_LISTEN_UDP_SOCKETS
249+
EOF
250+
fi
251+
252+
if [ "$SC4S_SOURCE_UDP_SO_RCVBUFF" -gt 0 ]; then
253+
cat >> "$OUTPUT_FILE" << EOF
254+
SC4S_SOURCE_UDP_SO_RCVBUFF=$SC4S_SOURCE_UDP_SO_RCVBUFF
255+
EOF
256+
fi
257+
258+
if [ "$SC4S_ENABLE_EBPF" == "yes" ]; then
259+
cat >> "$OUTPUT_FILE" << EOF
260+
SC4S_ENABLE_EBPF=$SC4S_ENABLE_EBPF
261+
SC4S_EBPF_NO_SOCKETS=$SC4S_EBPF_NO_SOCKETS
262+
EOF
263+
fi
264+
265+
cat >> "$OUTPUT_FILE" << EOF
266+
267+
EOF
268+
269+
if [ "$SC4S_SOURCE_TCP_SO_RCVBUFF" -gt 0 ]; then
270+
cat >> "$OUTPUT_FILE" << EOF
271+
SC4S_SOURCE_TCP_SO_RCVBUFF=$SC4S_SOURCE_TCP_SO_RCVBUFF
272+
EOF
273+
fi
274+
275+
if [ "$PARALLELIZE" == "yes" ]; then
276+
cat >> "$OUTPUT_FILE" << EOF
277+
SC4S_ENABLE_PARALLELIZE=yes
278+
SC4S_PARALLELIZE_NO_PARTITION=$SC4S_PARALLELIZE_NO_PARTITION
279+
EOF
280+
fi
281+
282+
if [ "$SC4S_SOURCE_TCP_IW_USE" == "yes" ]; then
283+
cat >> "$OUTPUT_FILE" << EOF
284+
SC4S_SOURCE_TCP_IW_SIZE=$SC4S_SOURCE_TCP_IW_SIZE
285+
EOF
286+
fi
287+
288+
# === Disk Buffer Configuration ===
289+
if [ "$ADJUST_DISKBUFF" == "yes" ]; then
290+
cat >> "$OUTPUT_FILE" << EOF
291+
292+
# === Disk buffer Configuration ===
293+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE
294+
EOF
295+
296+
if [ "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE" == "yes" ]; then
297+
cat >> "$OUTPUT_FILE" << EOF
298+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE
299+
EOF
300+
301+
if [ "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE" == "yes" ]; then
302+
cat >> "$OUTPUT_FILE" << EOF
303+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE
304+
EOF
305+
else
306+
cat >> "$OUTPUT_FILE" << EOF
307+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH
308+
EOF
309+
fi
310+
311+
cat >> "$OUTPUT_FILE" << EOF
312+
SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE
313+
EOF
314+
fi
315+
fi
316+
317+
echo ""
318+
echo "${GREEN}✓ Configuration generated successfully!${NC}"
319+
echo "File: ${YELLOW}$OUTPUT_FILE${NC}"
320+
echo ""
321+
echo "${BLUE}Configuration Summary:${NC}"
322+
echo " Splunk URL: $SPLUNK_URL"
323+
echo " Protocol: $PROTOCOL"
324+
echo " Expected EPS: $EXPECTED_EPS"
325+
echo ""
326+
echo "${GREEN}Configuration tool completed successfully!${NC}"
327+
328+
329+
# === Final recommendations ===
330+
if [[ "$SC4S_SOURCE_UDP_SO_RCVBUFF" -gt 0 || "$SC4S_SOURCE_TCP_SO_RCVBUFF" -gt 0 ]]; then
331+
echo ""
332+
echo "${YELLOW}Note: You may need to adjust your system's UDP/TCP receiving buffer settings to match the configured values.${NC}"
333+
echo "You can modify /etc/sysctl.conf following this documentation:"
334+
echo "https://splunk.github.io/splunk-connect-for-syslog/main/gettingstarted/getting-started-runtime-configuration/#tune-your-receive-buffer"
335+
fi
336+
337+
338+
if [ "$SC4S_ENABLE_EBPF" == "yes" ]; then
339+
echo ""
340+
echo "${YELLOW}Note: Enabling eBPF may require additional system permissions.${NC}"
341+
echo "Ensure that your system supports eBPF and that the necessary capabilities are granted to the SC4S process or container. Read more here: "
342+
echo "https://splunk.github.io/splunk-connect-for-syslog/main/configuration/#about-ebpf"
343+
fi

‎docs/architecture/index.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,25 @@ The following resources can help you determine the best protocol for your setup:
5454

5555
1. [Run performance tests for TCP](performance-tests.md#check-your-tcp-performance)
5656
2. [Run performance tests for UDP](performance-tests.md#check-your-udp-performance)
57+
58+
### Configuration helper for SC4S
59+
60+
SC4S includes an interactive configuration tool that simplifies the setup and optimization of your environment file. The tool walks you through key configuration decisions and generates a customized `env_file` based on your specific requirements.
61+
62+
**What the tool configures:**
63+
64+
* **Splunk HEC destination** - Connection URL, token, and TLS verification settings
65+
* **Protocol optimization** - Choose UDP, TCP, or both, with protocol-specific tuning options
66+
* **UDP settings** - Socket configuration, fetch limits, receive buffers, and eBPF support for high-throughput scenarios
67+
* **TCP settings** - Receive buffers, parallelization, and static window size tuning
68+
* **Disk buffering** - Local disk buffer settings to minimize data loss during connectivity issues
69+
70+
**To use the configuration tool:**
71+
72+
```bash
73+
sh ./configuration-tool.sh
74+
```
75+
76+
The tool will prompt you for your environment details and create an optimized `env_file` ready for deployment. This approach ensures consistent configuration and helps you apply best practices without manually editing environment variables.
77+
78+
You can also read about our [scaling solutions](performance-tests.md) and [configuration variables](../configuration.md).

‎mkdocs.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ nav:
7575
- SC4S Lite:
7676
- Intro: "lite.md"
7777
- Pluggable modules: "pluggable_modules.md"
78+
- Enterprise: enterprise.md
7879
- Edge Processor: "edge_processor.md"
7980
- Troubleshooting:
8081
- SC4S Startup and Validation: "troubleshooting/troubleshoot_SC4S_server.md"

0 commit comments

Comments
 (0)