diff --git a/terraform/modules/ecs/README.md b/terraform/modules/ecs/README.md index 806c553..23f0dcc 100644 --- a/terraform/modules/ecs/README.md +++ b/terraform/modules/ecs/README.md @@ -211,6 +211,7 @@ module "polytomic-ecs" { | [aws_ecs_service.sync](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource | | [aws_ecs_service.web](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource | | [aws_ecs_service.worker](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource | +| [aws_ecs_task_definition.ingest](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource | | [aws_ecs_task_definition.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource | | [aws_ecs_task_definition.schemacache](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource | | [aws_ecs_task_definition.stats_reporter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource | @@ -332,6 +333,8 @@ module "polytomic-ecs" { | [polytomic\_query\_runner\_exclude\_dbs](#input\_polytomic\_query\_runner\_exclude\_dbs) | List of databases to exclude from query runner | `list` | `[]` | no | | [polytomic\_query\_worker\_count](#input\_polytomic\_query\_worker\_count) | Number of query workers to use | `number` | `20` | no | | [polytomic\_record\_log\_disabled](#input\_polytomic\_record\_log\_disabled) | Globally disable record logging for this deployment | `bool` | `false` | no | +| [polytomic\_resource\_ingest\_cpu](#input\_polytomic\_resource\_ingest\_cpu) | CPU units for the ingest container | `number` | `2048` | no | +| [polytomic\_resource\_ingest\_memory](#input\_polytomic\_resource\_ingest\_memory) | Memory units for the ingest container | `number` | `8192` | no | | [polytomic\_resource\_scheduler\_cpu](#input\_polytomic\_resource\_scheduler\_cpu) | CPU units for the scheduler container | `number` | `1024` | no | | [polytomic\_resource\_scheduler\_memory](#input\_polytomic\_resource\_scheduler\_memory) | Memory units for the scheduler container | `number` | `2048` | no | | [polytomic\_resource\_schemacache\_cpu](#input\_polytomic\_resource\_schemacache\_cpu) | CPU units for the schemacache container | `number` | `2048` | no | diff --git a/terraform/modules/ecs/ecs-tasks.tf b/terraform/modules/ecs/ecs-tasks.tf index 5317634..f2822a5 100644 --- a/terraform/modules/ecs/ecs-tasks.tf +++ b/terraform/modules/ecs/ecs-tasks.tf @@ -69,6 +69,10 @@ resource "aws_ecs_task_definition" "worker" { merge(local.environment, { worker_log_group = module.ecs_log_groups["worker"].cloudwatch_log_group_name + env = merge(local.environment.env, { + INGEST_EXECUTOR_TASK_DEFINITION = aws_ecs_task_definition.ingest.arn, + INGEST_EXECUTOR_CONTAINER_NAME = "ingest" + }) } ) ) @@ -155,6 +159,10 @@ resource "aws_ecs_task_definition" "sync" { merge(local.environment, { sync_log_group = module.ecs_log_groups["sync"].cloudwatch_log_group_name + env = merge(local.environment.env, { + INGEST_EXECUTOR_TASK_DEFINITION = aws_ecs_task_definition.ingest.arn, + INGEST_EXECUTOR_CONTAINER_NAME = "ingest" + }) } ) ) @@ -202,6 +210,53 @@ resource "aws_ecs_task_definition" "scheduler" { merge(local.environment, { scheduler_log_group = module.ecs_log_groups["scheduler"].cloudwatch_log_group_name + env = merge(local.environment.env, { + INGEST_EXECUTOR_TASK_DEFINITION = aws_ecs_task_definition.ingest.arn, + INGEST_EXECUTOR_CONTAINER_NAME = "ingest" + }) + } + ) + ) + + volume { + name = "polytomic" + + efs_volume_configuration { + file_system_id = module.efs.id + root_directory = "/" + transit_encryption = "ENABLED" + transit_encryption_port = 2999 + } + } +} + +resource "aws_ecs_task_definition" "ingest" { + family = "${var.prefix}-ingest" + + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = var.polytomic_resource_ingest_cpu + memory = var.polytomic_resource_ingest_memory + + task_role_arn = aws_iam_role.polytomic_ecs_task_role.arn + execution_role_arn = aws_iam_role.polytomic_ecs_execution_role.arn + tags = merge( + var.tags, + { + Name = "${var.prefix}-ingest" + }) + + + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "X86_64" + } + + container_definitions = templatefile( + "${path.module}/task-definitions/ingest.json.tftpl", + merge(local.environment, + { + ingest_log_group = module.ecs_log_groups["ingest"].cloudwatch_log_group_name } ) ) diff --git a/terraform/modules/ecs/logs.tf b/terraform/modules/ecs/logs.tf index e3a082d..4649db7 100644 --- a/terraform/modules/ecs/logs.tf +++ b/terraform/modules/ecs/logs.tf @@ -18,7 +18,7 @@ module "ecs_log_groups" { source = "terraform-aws-modules/cloudwatch/aws//modules/log-group" version = "~> 3.0" - for_each = toset(["sync", "scheduler", "schemacache", "stats-reporter", "web", "worker"]) + for_each = toset(["sync", "scheduler", "schemacache", "stats-reporter", "web", "worker", "ingest"]) name = "${var.prefix}-${each.key}-logs" retention_in_days = var.log_retention_days diff --git a/terraform/modules/ecs/main.tf b/terraform/modules/ecs/main.tf index 94850c8..ea7f1f0 100644 --- a/terraform/modules/ecs/main.tf +++ b/terraform/modules/ecs/main.tf @@ -96,6 +96,7 @@ locals { worker_memory = var.polytomic_resource_worker_memory scheduler_memory = var.polytomic_resource_scheduler_memory schemacache_memory = var.polytomic_resource_schemacache_memory + ingest_memory = var.polytomic_resource_ingest_memory image = var.polytomic_image, region = var.region, polytomic_port = var.polytomic_port, diff --git a/terraform/modules/ecs/task-definitions/ingest.json.tftpl b/terraform/modules/ecs/task-definitions/ingest.json.tftpl new file mode 100644 index 0000000..1c65ab9 --- /dev/null +++ b/terraform/modules/ecs/task-definitions/ingest.json.tftpl @@ -0,0 +1,172 @@ +[{ + %{ if !polytomic_logger } + "logConfiguration": { + "logDriver": "awslogs", + "secretOptions": null, + "options": { + "awslogs-group": "${ingest_log_group}", + "awslogs-region": "${region}", + "awslogs-stream-prefix": "service" + } + }, + %{ else } + "logConfiguration": { + "logDriver": "splunk", + "options": { + "splunk-token": "test", + "splunk-url": "http://localhost:8000", + "splunk-verify-connection": "false" + } + }, + "dependsOn": [ + { + "containerName": "vector", + "condition": "HEALTHY" + } + ], + %{ endif } + "environment": [ + %{ for key, value in env ~} + { + "name": "${key}", + "value": "${value}" + }, + %{ endfor ~} + %{~ if polytomic_dd_agent } + { + "name": "DD_AGENT_HOST", + "value": "localhost" + }, + { + "name": "METRICS", + "value": "true" + }, + %{ endif ~} + { + "name": "ROLE", + "value": "ingest" + } + ], + "secrets": [ +%{ for key, _ in secrets ~} + { + "name": "${key}", + "valueFrom": "${task_secret_arn}:${key}::" + }%{ if key != keys(secrets)[length(keys(secrets)) - 1] },%{ endif } +%{ endfor ~} + ], + "mountPoints": [ + { + "containerPath": "${mount_path}", + "sourceVolume": "polytomic" + }], + "image": "${image}", + "portMappings": [ + { + "containerPort": 8126, + "hostPort": 8126 + } + ], + "name": "ingest", + "ulimits": [ + { + "name": "nofile", + "softLimit": 1024000, + "hardLimit": 1024000 + } + ] +} +%{~ if polytomic_logger }, +{ + "image": "${polytomic_logger_image}", + "logConfiguration": { + "logDriver": "awslogs", + "secretOptions": null, + "options": { + "awslogs-group": "${ingest_log_group}", + "awslogs-region": "${region}", + "awslogs-stream-prefix": "service" + } + }, + "name": "vector", + "essential": true, + "environment": [ + %{ for key, value in env ~} + { + "name": "${key}", + "value": "${value}" + }, + %{ endfor ~} + { + "name": "ROLE", + "value": "logger" + } + ], + "secrets": [ +%{ for key, _ in support_secrets ~} + { + "name": "${key}", + "valueFrom": "${task_secret_arn}:${key}::" + }%{ if key != keys(support_secrets)[length(keys(support_secrets)) - 1] },%{ endif } +%{ endfor ~} + ], + "healthCheck": { + "command": [ + "CMD-SHELL", + "wget --no-verbose --tries=1 --spider http://localhost:8000/services/collector/health > /dev/null || exit 1" + ], + "interval":30, + "retries":3, + "startPeriod":100, + "timeout":5 + } +} + %{ endif } + %{~ if polytomic_dd_agent }, +{ + "name": "datadog-agent", + "image": "${polytomic_dd_agent_image}", + "essential": true, + "environment": [ + %{ for key, value in env ~} + { + "name": "${key}", + "value": "${value}" + }, + %{ endfor ~} + { + "name": "DD_API_KEY", + "value": "ENC[DD_API_KEY]" + }, + { + "name": "DD_SECRET_BACKEND_COMMAND", + "value": "/bin/ptctl" + }, + { + "name": "DD_SECRET_BACKEND_ARGUMENTS", + "value": "config get -f vector -" + }, + { + "name": "DD_DOCKER_ENV_AS_TAGS", + "value": "{\\\"role\\\": \\\"polytomic.role\\\"}" + }, + { + "name": "DD_APM_ENABLED", + "value": "true" + }, + { + "name": "ECS_FARGATE", + "value": "true" + } + ], + "secrets": [ +%{ for key, _ in support_secrets ~} + { + "name": "${key}", + "valueFrom": "${task_secret_arn}:${key}::" + }%{ if key != keys(support_secrets)[length(keys(support_secrets)) - 1] },%{ endif } +%{ endfor ~} + ] +} + %{ endif } +] diff --git a/terraform/modules/ecs/vars.tf b/terraform/modules/ecs/vars.tf index eb460ab..c2fa60d 100644 --- a/terraform/modules/ecs/vars.tf +++ b/terraform/modules/ecs/vars.tf @@ -187,6 +187,16 @@ variable "polytomic_resource_sync_storage" { default = 100 // 100 GB } +variable "polytomic_resource_ingest_cpu" { + description = "CPU units for the ingest container" + default = 2048 // 2 vCPU +} + +variable "polytomic_resource_ingest_memory" { + description = "Memory units for the ingest container" + default = 8192 // 8 GB +} + variable "polytomic_mssql_tx_isolation" { description = "Transaction isolation level for MSSQL connections" default = ""