diff --git a/terraform/modules/ecs/README.md b/terraform/modules/ecs/README.md
index 806c553..23f0dcc 100644
--- a/terraform/modules/ecs/README.md
+++ b/terraform/modules/ecs/README.md
@@ -211,6 +211,7 @@ module "polytomic-ecs" {
| [aws_ecs_service.sync](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
| [aws_ecs_service.web](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
| [aws_ecs_service.worker](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
+| [aws_ecs_task_definition.ingest](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
| [aws_ecs_task_definition.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
| [aws_ecs_task_definition.schemacache](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
| [aws_ecs_task_definition.stats_reporter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
@@ -332,6 +333,8 @@ module "polytomic-ecs" {
| [polytomic\_query\_runner\_exclude\_dbs](#input\_polytomic\_query\_runner\_exclude\_dbs) | List of databases to exclude from query runner | `list` | `[]` | no |
| [polytomic\_query\_worker\_count](#input\_polytomic\_query\_worker\_count) | Number of query workers to use | `number` | `20` | no |
| [polytomic\_record\_log\_disabled](#input\_polytomic\_record\_log\_disabled) | Globally disable record logging for this deployment | `bool` | `false` | no |
+| [polytomic\_resource\_ingest\_cpu](#input\_polytomic\_resource\_ingest\_cpu) | CPU units for the ingest container | `number` | `2048` | no |
+| [polytomic\_resource\_ingest\_memory](#input\_polytomic\_resource\_ingest\_memory) | Memory units for the ingest container | `number` | `8192` | no |
| [polytomic\_resource\_scheduler\_cpu](#input\_polytomic\_resource\_scheduler\_cpu) | CPU units for the scheduler container | `number` | `1024` | no |
| [polytomic\_resource\_scheduler\_memory](#input\_polytomic\_resource\_scheduler\_memory) | Memory units for the scheduler container | `number` | `2048` | no |
| [polytomic\_resource\_schemacache\_cpu](#input\_polytomic\_resource\_schemacache\_cpu) | CPU units for the schemacache container | `number` | `2048` | no |
diff --git a/terraform/modules/ecs/ecs-tasks.tf b/terraform/modules/ecs/ecs-tasks.tf
index 5317634..f2822a5 100644
--- a/terraform/modules/ecs/ecs-tasks.tf
+++ b/terraform/modules/ecs/ecs-tasks.tf
@@ -69,6 +69,10 @@ resource "aws_ecs_task_definition" "worker" {
merge(local.environment,
{
worker_log_group = module.ecs_log_groups["worker"].cloudwatch_log_group_name
+ env = merge(local.environment.env, {
+ INGEST_EXECUTOR_TASK_DEFINITION = aws_ecs_task_definition.ingest.arn,
+ INGEST_EXECUTOR_CONTAINER_NAME = "ingest"
+ })
}
)
)
@@ -155,6 +159,10 @@ resource "aws_ecs_task_definition" "sync" {
merge(local.environment,
{
sync_log_group = module.ecs_log_groups["sync"].cloudwatch_log_group_name
+ env = merge(local.environment.env, {
+ INGEST_EXECUTOR_TASK_DEFINITION = aws_ecs_task_definition.ingest.arn,
+ INGEST_EXECUTOR_CONTAINER_NAME = "ingest"
+ })
}
)
)
@@ -202,6 +210,53 @@ resource "aws_ecs_task_definition" "scheduler" {
merge(local.environment,
{
scheduler_log_group = module.ecs_log_groups["scheduler"].cloudwatch_log_group_name
+ env = merge(local.environment.env, {
+ INGEST_EXECUTOR_TASK_DEFINITION = aws_ecs_task_definition.ingest.arn,
+ INGEST_EXECUTOR_CONTAINER_NAME = "ingest"
+ })
+ }
+ )
+ )
+
+ volume {
+ name = "polytomic"
+
+ efs_volume_configuration {
+ file_system_id = module.efs.id
+ root_directory = "/"
+ transit_encryption = "ENABLED"
+ transit_encryption_port = 2999
+ }
+ }
+}
+
+resource "aws_ecs_task_definition" "ingest" {
+ family = "${var.prefix}-ingest"
+
+ requires_compatibilities = ["FARGATE"]
+ network_mode = "awsvpc"
+ cpu = var.polytomic_resource_ingest_cpu
+ memory = var.polytomic_resource_ingest_memory
+
+ task_role_arn = aws_iam_role.polytomic_ecs_task_role.arn
+ execution_role_arn = aws_iam_role.polytomic_ecs_execution_role.arn
+ tags = merge(
+ var.tags,
+ {
+ Name = "${var.prefix}-ingest"
+ })
+
+
+ runtime_platform {
+ operating_system_family = "LINUX"
+ cpu_architecture = "X86_64"
+ }
+
+ container_definitions = templatefile(
+ "${path.module}/task-definitions/ingest.json.tftpl",
+ merge(local.environment,
+ {
+ ingest_log_group = module.ecs_log_groups["ingest"].cloudwatch_log_group_name
}
)
)
diff --git a/terraform/modules/ecs/logs.tf b/terraform/modules/ecs/logs.tf
index e3a082d..4649db7 100644
--- a/terraform/modules/ecs/logs.tf
+++ b/terraform/modules/ecs/logs.tf
@@ -18,7 +18,7 @@ module "ecs_log_groups" {
source = "terraform-aws-modules/cloudwatch/aws//modules/log-group"
version = "~> 3.0"
- for_each = toset(["sync", "scheduler", "schemacache", "stats-reporter", "web", "worker"])
+ for_each = toset(["sync", "scheduler", "schemacache", "stats-reporter", "web", "worker", "ingest"])
name = "${var.prefix}-${each.key}-logs"
retention_in_days = var.log_retention_days
diff --git a/terraform/modules/ecs/main.tf b/terraform/modules/ecs/main.tf
index 94850c8..ea7f1f0 100644
--- a/terraform/modules/ecs/main.tf
+++ b/terraform/modules/ecs/main.tf
@@ -96,6 +96,7 @@ locals {
worker_memory = var.polytomic_resource_worker_memory
scheduler_memory = var.polytomic_resource_scheduler_memory
schemacache_memory = var.polytomic_resource_schemacache_memory
+ ingest_memory = var.polytomic_resource_ingest_memory
image = var.polytomic_image,
region = var.region,
polytomic_port = var.polytomic_port,
diff --git a/terraform/modules/ecs/task-definitions/ingest.json.tftpl b/terraform/modules/ecs/task-definitions/ingest.json.tftpl
new file mode 100644
index 0000000..1c65ab9
--- /dev/null
+++ b/terraform/modules/ecs/task-definitions/ingest.json.tftpl
@@ -0,0 +1,172 @@
+[{
+ %{ if !polytomic_logger }
+ "logConfiguration": {
+ "logDriver": "awslogs",
+ "secretOptions": null,
+ "options": {
+ "awslogs-group": "${ingest_log_group}",
+ "awslogs-region": "${region}",
+ "awslogs-stream-prefix": "service"
+ }
+ },
+ %{ else }
+ "logConfiguration": {
+ "logDriver": "splunk",
+ "options": {
+ "splunk-token": "test",
+ "splunk-url": "http://localhost:8000",
+ "splunk-verify-connection": "false"
+ }
+ },
+ "dependsOn": [
+ {
+ "containerName": "vector",
+ "condition": "HEALTHY"
+ }
+ ],
+ %{ endif }
+ "environment": [
+ %{ for key, value in env ~}
+ {
+ "name": "${key}",
+ "value": "${value}"
+ },
+ %{ endfor ~}
+ %{~ if polytomic_dd_agent }
+ {
+ "name": "DD_AGENT_HOST",
+ "value": "localhost"
+ },
+ {
+ "name": "METRICS",
+ "value": "true"
+ },
+ %{ endif ~}
+ {
+ "name": "ROLE",
+ "value": "ingest"
+ }
+ ],
+ "secrets": [
+%{ for key, _ in secrets ~}
+ {
+ "name": "${key}",
+ "valueFrom": "${task_secret_arn}:${key}::"
+ }%{ if key != keys(secrets)[length(keys(secrets)) - 1] },%{ endif }
+%{ endfor ~}
+ ],
+ "mountPoints": [
+ {
+ "containerPath": "${mount_path}",
+ "sourceVolume": "polytomic"
+ }],
+ "image": "${image}",
+ "portMappings": [
+ {
+ "containerPort": 8126,
+ "hostPort": 8126
+ }
+ ],
+ "name": "ingest",
+ "ulimits": [
+ {
+ "name": "nofile",
+ "softLimit": 1024000,
+ "hardLimit": 1024000
+ }
+ ]
+}
+%{~ if polytomic_logger },
+{
+ "image": "${polytomic_logger_image}",
+ "logConfiguration": {
+ "logDriver": "awslogs",
+ "secretOptions": null,
+ "options": {
+ "awslogs-group": "${ingest_log_group}",
+ "awslogs-region": "${region}",
+ "awslogs-stream-prefix": "service"
+ }
+ },
+ "name": "vector",
+ "essential": true,
+ "environment": [
+ %{ for key, value in env ~}
+ {
+ "name": "${key}",
+ "value": "${value}"
+ },
+ %{ endfor ~}
+ {
+ "name": "ROLE",
+ "value": "logger"
+ }
+ ],
+ "secrets": [
+%{ for key, _ in support_secrets ~}
+ {
+ "name": "${key}",
+ "valueFrom": "${task_secret_arn}:${key}::"
+ }%{ if key != keys(support_secrets)[length(keys(support_secrets)) - 1] },%{ endif }
+%{ endfor ~}
+ ],
+ "healthCheck": {
+ "command": [
+ "CMD-SHELL",
+ "wget --no-verbose --tries=1 --spider http://localhost:8000/services/collector/health > /dev/null || exit 1"
+ ],
+ "interval":30,
+ "retries":3,
+ "startPeriod":100,
+ "timeout":5
+ }
+}
+ %{ endif }
+ %{~ if polytomic_dd_agent },
+{
+ "name": "datadog-agent",
+ "image": "${polytomic_dd_agent_image}",
+ "essential": true,
+ "environment": [
+ %{ for key, value in env ~}
+ {
+ "name": "${key}",
+ "value": "${value}"
+ },
+ %{ endfor ~}
+ {
+ "name": "DD_API_KEY",
+ "value": "ENC[DD_API_KEY]"
+ },
+ {
+ "name": "DD_SECRET_BACKEND_COMMAND",
+ "value": "/bin/ptctl"
+ },
+ {
+ "name": "DD_SECRET_BACKEND_ARGUMENTS",
+ "value": "config get -f vector -"
+ },
+ {
+ "name": "DD_DOCKER_ENV_AS_TAGS",
+ "value": "{\\\"role\\\": \\\"polytomic.role\\\"}"
+ },
+ {
+ "name": "DD_APM_ENABLED",
+ "value": "true"
+ },
+ {
+ "name": "ECS_FARGATE",
+ "value": "true"
+ }
+ ],
+ "secrets": [
+%{ for key, _ in support_secrets ~}
+ {
+ "name": "${key}",
+ "valueFrom": "${task_secret_arn}:${key}::"
+ }%{ if key != keys(support_secrets)[length(keys(support_secrets)) - 1] },%{ endif }
+%{ endfor ~}
+ ]
+}
+ %{ endif }
+]
diff --git a/terraform/modules/ecs/vars.tf b/terraform/modules/ecs/vars.tf
index eb460ab..c2fa60d 100644
--- a/terraform/modules/ecs/vars.tf
+++ b/terraform/modules/ecs/vars.tf
@@ -187,6 +187,16 @@ variable "polytomic_resource_sync_storage" {
default = 100 // 100 GB
}
+variable "polytomic_resource_ingest_cpu" {
+ description = "CPU units for the ingest container"
+ default = 2048 // 2 vCPU
+}
+
+variable "polytomic_resource_ingest_memory" {
+ description = "Memory units for the ingest container"
+ default = 8192 // 8 GB
+}
+
variable "polytomic_mssql_tx_isolation" {
description = "Transaction isolation level for MSSQL connections"
default = ""