Skip to content

Commit 23b2865

Browse files
cybercystcybercyst
authored
Merge pull request #1 from docker/ip-tagging-async-file-reload
Ip tagging async file reload
2 parents 5795d2a + 75cb494 commit 23b2865

32 files changed

+1911
-209
lines changed

‎api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ syntax = "proto3";
33
package envoy.extensions.filters.http.ip_tagging.v3;
44

55
import "envoy/config/core/v3/address.proto";
6+
import "envoy/config/core/v3/base.proto";
7+
8+
import "google/protobuf/duration.proto";
69

710
import "udpa/annotations/status.proto";
811
import "udpa/annotations/versioning.proto";
@@ -18,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
1821
// IP tagging :ref:`configuration overview <config_http_filters_ip_tagging>`.
1922
// [#extension: envoy.filters.http.ip_tagging]
2023

21-
// [#next-free-field: 6]
24+
// [#next-free-field: 7]
2225
message IPTagging {
2326
option (udpa.annotations.versioning).previous_message_type =
2427
"envoy.config.filter.http.ip_tagging.v2.IPTagging";
@@ -53,6 +56,12 @@ message IPTagging {
5356
repeated config.core.v3.CidrRange ip_list = 2;
5457
}
5558

59+
// Specifies the content of the IP tag file.
60+
// Allow the file to be created with no IP tags.
61+
message IPTags {
62+
repeated IPTag ip_tags = 1;
63+
}
64+
5665
// Specify to which header the tags will be written.
5766
message IpTagHeader {
5867
// Describes how to apply the tags to the headers.
@@ -85,16 +94,33 @@ message IPTagging {
8594
HeaderAction action = 2;
8695
}
8796

97+
// Common configuration for file based ip tags.
98+
message IpTagsFileProvider {
99+
// Data source from which to retrieve ip tags.
100+
// Only filename based data source is currently supported for IP tags.
101+
// When using this data source, if a ``watched_directory`` is provided, the IP tags file will be re-read when a file move is detected.
102+
// See :ref:`watched_directory <envoy_v3_api_msg_config.core.v3.DataSource>` for more information about the ``watched_directory`` field.
103+
config.core.v3.DataSource ip_tags_datasource = 1;
104+
105+
// When :ref:`ip_tags <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_datasource>` is configured
106+
// the ip tags will be reloaded from file every ``ip_tags_refresh_rate`` milliseconds.
107+
// Defaults to 0, in this case no refresh will be attempted.
108+
google.protobuf.Duration ip_tags_refresh_rate = 2 [(validate.rules).duration = {gt {}}];
109+
}
110+
88111
// The type of request the filter should apply to.
89112
RequestType request_type = 1 [(validate.rules).enum = {defined_only: true}];
90113

91-
// [#comment:TODO(ccaraman): Extend functionality to load IP tags from file system.
92-
// Tracked by issue https://github.com/envoyproxy/envoy/issues/2695]
93114
// The set of IP tags for the filter.
94-
repeated IPTag ip_tags = 4 [(validate.rules).repeated = {min_items: 1}];
115+
// Only one of :ref:`ip_tags <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.ip_tags>`
116+
// or :ref:`ip_tags_datasource <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_datasource>`
117+
// can be set for the IP Tagging filter.
118+
repeated IPTag ip_tags = 4;
95119

96120
// Specify to which header the tags will be written.
97121
//
98122
// If left unspecified, the tags will be appended to the ``x-envoy-ip-tags`` header.
99123
IpTagHeader ip_tag_header = 5;
124+
125+
IpTagsFileProvider ip_tags_file_provider = 6;
100126
}

‎docs/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ filegroup(
3434
"root/configuration/listeners/network_filters/_include/generic_proxy_filter.yaml",
3535
"root/configuration/overview/_include/xds_api/oauth-sds-example.yaml",
3636
"root/configuration/security/_include/sds-source-example.yaml",
37+
"root/configuration/http/http_filters/_include/ip-tagging-filter.yaml",
3738
],
3839
) + select({
3940
"//bazel:windows_x86_64": [],

‎docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
static_resources:
2+
listeners:
3+
- address:
4+
socket_address:
5+
address: 0.0.0.0
6+
port_value: 8000
7+
filter_chains:
8+
- filters:
9+
- name: envoy.filters.network.http_connection_manager
10+
typed_config:
11+
'@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
12+
stat_prefix: ingress_http
13+
http_filters:
14+
- name: ip_tagging
15+
typed_config:
16+
"@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging
17+
request_type: both
18+
ip_tags:
19+
- ip_tag_name: external_request
20+
ip_list:
21+
- {address_prefix: 1.2.3.4, prefix_len: 32}
22+
- name: envoy.filters.http.router
23+
typed_config:
24+
'@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
25+
route_config:
26+
name: local_route
27+
virtual_hosts:
28+
- domains:
29+
- '*'
30+
name: local_service
31+
routes:
32+
- match: {prefix: "/"}
33+
route: {cluster: default_service}
34+
- address:
35+
socket_address:
36+
address: 0.0.0.0
37+
port_value: 9000
38+
filter_chains:
39+
- filters:
40+
- name: envoy.filters.network.http_connection_manager
41+
typed_config:
42+
'@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
43+
stat_prefix: ingress1_http
44+
http_filters:
45+
- name: ip_tagging
46+
typed_config:
47+
"@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging
48+
request_type: both
49+
ip_tags_file_provider:
50+
ip_tags_refresh_rate: 5s
51+
ip_tags_datasource:
52+
filename: "/geoip/ip-tags.yaml"
53+
watched_directory:
54+
path: "/geoip/"
55+
- name: envoy.filters.http.router
56+
typed_config:
57+
'@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
58+
route_config:
59+
name: local_route
60+
virtual_hosts:
61+
- domains:
62+
- '*'
63+
name: local_service
64+
routes:
65+
- match: {prefix: "/"}
66+
route: {cluster: default_service}
67+
- address:
68+
socket_address:
69+
address: 0.0.0.0
70+
port_value: 7000
71+
filter_chains:
72+
- filters:
73+
- name: envoy.filters.network.http_connection_manager
74+
typed_config:
75+
'@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
76+
stat_prefix: ingress2_http
77+
http_filters:
78+
- name: ip_tagging
79+
typed_config:
80+
"@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging
81+
request_type: both
82+
ip_tags_file_provider:
83+
ip_tags_refresh_rate: 5s
84+
ip_tags_datasource:
85+
filename: "/geoip/ip-tags.json"
86+
watched_directory:
87+
path: "/geoip/"
88+
- name: envoy.filters.http.router
89+
typed_config:
90+
'@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
91+
route_config:
92+
name: local_route
93+
virtual_hosts:
94+
- domains:
95+
- '*'
96+
name: local_service
97+
routes:
98+
- match: {prefix: "/"}
99+
route: {cluster: default_service}
100+
clusters:
101+
- name: default_service
102+
load_assignment:
103+
cluster_name: default_service
104+
endpoints:
105+
- lb_endpoints:
106+
- endpoint:
107+
address:
108+
socket_address:
109+
address: 127.0.0.1
110+
port_value: 10001
111+
admin:
112+
address:
113+
socket_address:
114+
address: 0.0.0.0
115+
port_value: 9901

‎docs/root/configuration/http/http_filters/ip_tagging_filter.rst‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,83 @@ described in the paper `IP-address lookup using
2222
LC-tries <https://www.csc.kth.se/~snilsson/publications/IP-address-lookup-using-LC-tries/text.pdf>`_ by S. Nilsson and
2323
G. Karlsson.
2424

25+
IP tags can either be provided directly using the :ref:`ip_tags <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.ip_tags>` API field or
26+
can be loaded from file if :ref:`ip_tags_datasource <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_datasource>` API field is configured.
27+
For file based IP tags YAML and JSON file formats are supported.
28+
IP tags will be dynamically reloaded if ``watched_directory`` is configured for :ref:`ip_tags_datasource <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_datasource>`
29+
and :ref:`ip_tags_refresh_rate <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_refresh_rate>` is set to value greater than zero.
2530

2631
Configuration
2732
-------------
2833
* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging``.
2934
* :ref:`v3 API reference <envoy_v3_api_msg_extensions.filters.http.ip_tagging.v3.IPTagging>`
3035

36+
An example configuration of the filter with inline ip tags may look like the following:
37+
38+
.. literalinclude:: _include/ip-tagging-filter.yaml
39+
:language: yaml
40+
:lines: 13-21
41+
:lineno-start: 13
42+
:linenos:
43+
:caption: :download:`ip-tagging-filter.yaml <_include/ip-tagging-filter.yaml>`
44+
45+
Below is an example configuration of the filter with the file based ip tags in yaml format:
46+
47+
.. literalinclude:: _include/ip-tagging-filter.yaml
48+
:language: yaml
49+
:lines: 44-54
50+
:lineno-start: 44
51+
:linenos:
52+
:caption: :download:`ip-tagging-filter.yaml <_include/ip-tagging-filter.yaml>`
53+
54+
Where the *ip-tags.yaml* file would have the following content:
55+
56+
.. code-block:: yaml
57+
58+
ip_tags:
59+
- ip_tag_name: external_request
60+
ip_list:
61+
- {address_prefix: 1.2.3.4, prefix_len: 32}
62+
- ip_tag_name: internal_request
63+
ip_list:
64+
- {address_prefix: 1.2.3.5, prefix_len: 32}
65+
66+
And here is an example configuration of the filter with the file based IP tags in JSON format:
67+
68+
.. literalinclude:: _include/ip-tagging-filter.yaml
69+
:language: yaml
70+
:lines: 77-87
71+
:lineno-start: 77
72+
:linenos:
73+
:caption: :download:`ip-tagging-filter.yaml <_include/ip-tagging-filter.yaml>`
74+
75+
Where the ``ip-tags.json`` file would have the following content:
76+
77+
.. code-block:: json
78+
79+
{
80+
"ip_tags": [
81+
{
82+
"ip_tag_name": "external_request",
83+
"ip_list": [
84+
{
85+
"address_prefix": "1.2.3.4",
86+
"prefix_len": 32
87+
}
88+
]
89+
},
90+
{
91+
"ip_tag_name": "internal_request",
92+
"ip_list": [
93+
{
94+
"address_prefix": "1.2.3.5",
95+
"prefix_len": 32
96+
}
97+
]
98+
}
99+
]
100+
}
101+
31102
Statistics
32103
----------
33104

@@ -41,6 +112,8 @@ the owning HTTP connection manager.
41112
<tag_name>.hit, Counter, Total number of requests that have the ``<tag_name>`` applied to it
42113
no_hit, Counter, Total number of requests with no applicable IP tags
43114
total, Counter, Total number of requests the IP Tagging Filter operated on
115+
reload_success, Counter, Total number of successful reloads of IP tags file
116+
reload_error, Counter, Total number of failed reloads of IP tags file
44117

45118
Runtime
46119
-------

‎source/common/common/logger.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ const static bool should_log = true;
101101
FUNCTION(websocket) \
102102
FUNCTION(golang) \
103103
FUNCTION(stats_sinks) \
104-
FUNCTION(dynamic_modules)
104+
FUNCTION(dynamic_modules) \
105+
FUNCTION(ip_tagging)
105106

106107
// clang-format off
107108
enum class Id {

‎source/common/protobuf/utility.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,8 @@ class MessageUtil {
271271
static void loadFromJson(absl::string_view json, Protobuf::Struct& message);
272272
static void loadFromYaml(const std::string& yaml, Protobuf::Message& message,
273273
ProtobufMessage::ValidationVisitor& validation_visitor);
274+
static absl::Status loadFromYamlNoThrow(const std::string& yaml, Protobuf::Message& message,
275+
ProtobufMessage::ValidationVisitor& validation_visitor);
274276
#endif
275277

276278
// This function attempts to load Envoy configuration from the specified file

‎source/common/protobuf/yaml_utility.cc‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,20 @@ void jsonConvertInternal(const Protobuf::Message& source,
113113
MessageUtil::loadFromJson(json, dest, validation_visitor);
114114
}
115115

116+
absl::Status jsonConvertInternalNoThrow(const Protobuf::Message& source,
117+
ProtobufMessage::ValidationVisitor& validation_visitor,
118+
Protobuf::Message& dest) {
119+
absl::Status conversion_status = absl::OkStatus();
120+
Protobuf::util::JsonPrintOptions json_options;
121+
json_options.preserve_proto_field_names = true;
122+
std::string json;
123+
conversion_status = Protobuf::util::MessageToJsonString(source, &json, json_options);
124+
if (conversion_status.ok()) {
125+
MessageUtil::loadFromJson(json, dest, validation_visitor);
126+
}
127+
return conversion_status;
128+
}
129+
116130
} // namespace
117131

118132
void MessageUtil::loadFromJson(absl::string_view json, Protobuf::Message& message,
@@ -181,6 +195,38 @@ void MessageUtil::loadFromYaml(const std::string& yaml, Protobuf::Message& messa
181195
throw EnvoyException("Unable to convert YAML as JSON: " + yaml);
182196
}
183197

198+
absl::Status
199+
MessageUtil::loadFromYamlNoThrow(const std::string& yaml, Protobuf::Message& message,
200+
ProtobufMessage::ValidationVisitor& validation_visitor) {
201+
absl::Status load_status = absl::OkStatus();
202+
Protobuf::Value value;
203+
TRY_NEEDS_AUDIT { value = parseYamlNode(YAML::Load(yaml)); }
204+
END_TRY
205+
catch (YAML::ParserException& e) {
206+
load_status = absl::InvalidArgumentError(fmt::format("Failed to parse yaml: {}", e.what()));
207+
}
208+
catch (YAML::BadConversion& e) {
209+
load_status =
210+
absl::InvalidArgumentError(fmt::format("Failed to convert to yaml: {}", e.what()));
211+
}
212+
catch (std::exception& e) {
213+
// There is a potentially wide space of exceptions thrown by the YAML parser,
214+
// and enumerating them all may be difficult. Envoy doesn't work well with
215+
// unhandled exceptions, so we capture them and record the exception name in
216+
// the status.
217+
load_status =
218+
absl::InvalidArgumentError(fmt::format("Unexpected YAML exception: {}", e.what()));
219+
}
220+
if (value.kind_case() == Protobuf::Value::kStructValue ||
221+
value.kind_case() == Protobuf::Value::kListValue) {
222+
load_status = jsonConvertInternalNoThrow(value, validation_visitor, message);
223+
} else {
224+
load_status =
225+
absl::InvalidArgumentError(fmt::format("Unable to convert YAML as JSON: {}", yaml));
226+
}
227+
return load_status;
228+
}
229+
184230
std::string MessageUtil::getYamlStringFromMessage(const Protobuf::Message& message,
185231
const bool block_print,
186232
const bool always_print_primitive_fields) {

‎source/extensions/filters/http/ip_tagging/BUILD‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@ envoy_cc_library(
1919
deps = [
2020
"//envoy/http:filter_interface",
2121
"//envoy/runtime:runtime_interface",
22+
"//envoy/thread_local:thread_local_interface",
2223
"//source/common/common:assert_lib",
24+
"//source/common/common:thread_synchronizer_lib",
25+
"//source/common/config:datasource_lib",
2326
"//source/common/http:header_map_lib",
2427
"//source/common/http:headers_lib",
2528
"//source/common/network:lc_trie_lib",
@@ -35,6 +38,7 @@ envoy_cc_extension(
3538
hdrs = ["config.h"],
3639
deps = [
3740
"//envoy/registry",
41+
"//envoy/thread_local:thread_local_interface",
3842
"//source/common/protobuf:utility_lib",
3943
"//source/extensions/filters/http/common:factory_base_lib",
4044
"//source/extensions/filters/http/ip_tagging:ip_tagging_filter_lib",

0 commit comments

Comments
 (0)