Terraform timeout on creating autoscaling group with an initial lifecycle hook declaration

2,220 views

Skip to first unread message

CheeWee Chua

unread,

Sep 17, 2018, 12:24:24 AM9/17/18

to Terraform

Hi,

I have the following Terraform configuration.

I'm trying to attach a lifecycle hook to the autoscaling group.

With the lifecycle hook declared within the autoscaling group, Terraform takes about 10 minutes to try and create the autoscaling group, before it times out.

I go to the AWS console, and I see the autoscaling group has been created, but with both min,max=0 (however, my configuration sets min=1, max=4).

I then get the following error in Terraform

* aws_autoscaling_group.chuacw102: 1 error(s) occurred:

* aws_autoscaling_group.chuacw102: "tf-asg-20180917040621577300000008": Waiting up to 10m0s: Need at least 1 healthy instances in ASG, have 0. Most recent activity: {
ActivityId: "6b859796-d557-d3d3-796d-90a1065b059f",
AutoScalingGroupName: "tf-asg-20180917040621577300000008",
Cause: "At 2018-09-17T04:06:27Z a user request update of AutoScalingGroup constraints to min: 1, max: 4, desired: 1 changing the desired capacity from 0 to 1. At 2018-09-17T04:06:29Z an instance was started in response to a difference between desired and actual capacity, increasing the capacity from 0 to 1.",
Description: "Launching a new EC2 instance: i-008d7288404e36efd",
Details: "{\"Availability Zone\":\"us-east-1a\"}",
Progress: 40,
StartTime: 2018-09-17 04:06:31.156 +0000 UTC,
StatusCode: "MidLifecycleAction"
}

If I move the lifecycle hook outside of the autoscaling group, everything gets created successfully, but my lifecycle hook misses the first instance that was created by the autoscaling group.

Any one have any ideas how I can keep the lifecycle hook within the autoscaling group and not have the above error?

Thank you.

provider "aws" {
    region = "us-east-1"
}

provider "archive" {}

data "archive_file" "zip" {
  type        = "zip"
  source_file = "../../BackupLambda"
  output_path = "BackupLambda.zip"
}

# Declare the lambda function that will be launched
resource "aws_lambda_function" "lambda_code102" {
  filename         = "${data.archive_file.zip.output_path}"
  function_name    = "NotifyAutoscalingGroupLaunched"
  role             = "${aws_iam_role.iam_for_lambda102.arn}"
  handler          = "BackupLambda" # Name of Go package after unzipping the filename above
  source_code_hash = "${data.archive_file.zip.output_sha}" # 
  runtime          = "go1.x"
}

resource "aws_sns_topic" "topic102" {
  name = "topic102"
}

resource "aws_sns_topic_subscription" "sms102" {
  topic_arn = "${aws_sns_topic.topic102.arn}"
  protocol  = "sms"
  endpoint  = "xxxxxxxx" // obfuscated phone number
}

resource "aws_sns_topic_subscription" "lambda102" {
  topic_arn = "${aws_sns_topic.topic102.arn}"
  protocol  = "lambda"
  endpoint  = "${aws_lambda_function.lambda_code102.arn}"
}

resource "aws_lambda_permission" "with_sns102-01" {
    statement_id = "AllowExecutionFromSNS102"
    action = "lambda:InvokeFunction"
    function_name = "${aws_lambda_function.lambda_code102.arn}"
    principal = "sns.amazonaws.com"
    source_arn = "${aws_sns_topic.topic102.arn}"
}

resource "aws_lambda_permission" "with_sns102-02" {
    statement_id = "AllowExecutionFromAutoscaling102"
    action = "lambda:InvokeFunction"
    function_name = "${aws_lambda_function.lambda_code102.arn}"
    principal = "autoscaling.amazonaws.com"
    source_arn = "${aws_sns_topic.topic102.arn}"
}

resource "aws_lambda_permission" "with_sns102-03" {
    statement_id = "AllowExecutionFromAutoscalingGroup102-01"
    action = "lambda:InvokeFunction"
    function_name = "${aws_lambda_function.lambda_code102.arn}"
    principal = "events.amazonaws.com"
    source_arn = "${aws_sns_topic.topic102.arn}"
}

resource "aws_lambda_permission" "with_sns102-04" {
    statement_id = "AllowExecutionFromAutoscalingGroup102-02"
    action = "lambda:InvokeFunction"
    function_name = "${aws_lambda_function.lambda_code102.arn}"
    principal = "autoscaling.amazonaws.com"
    source_arn = "${aws_autoscaling_group.chuacw102.arn}"
}

# receiving notification from autoscaling events #1
resource "aws_autoscaling_notification" "chuacw-notif102-01" {
  group_names = [
    "${aws_autoscaling_group.chuacw102.name}"
  ]
    
  notifications = [
    "autoscaling:EC2_INSTANCE_LAUNCH",
    "autoscaling:TEST_NOTIFICATION"
  ]

  topic_arn = "${aws_sns_topic.topic102.arn}"
}

resource "aws_cloudwatch_event_rule" "autoscaling102" {
  name        = "capture-autoscaling-102"
  description = "Capture all autoscaling events"

  event_pattern = <<PATTERN
{
  "source": [
    "aws.autoscaling"
  ]
}
PATTERN
}

resource "aws_cloudwatch_event_target" "sns102" {
  rule      = "${aws_cloudwatch_event_rule.autoscaling102.name}"
  target_id = "SendToSNS102"
  arn       = "${aws_sns_topic.topic102.arn}"
}

resource "aws_cloudwatch_event_target" "lambda102" {
  rule      = "${aws_cloudwatch_event_rule.autoscaling102.name}"
  target_id = "SendToLambda102"
  arn       = "${aws_lambda_function.lambda_code102.arn}"
}

resource "aws_iam_role" "iam_for_lambda102" {
  name = "iam_for_lambda"
# See also https://aws.amazon.com/blogs/compute/easy-authorization-of-aws-lambda-functions/
  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "lambda.amazonaws.com",
        "Service": "autoscaling.amazonaws.com",
        "Service": "events.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "sns_access102" {
   role = "${aws_iam_role.iam_for_lambda102.name}"
   policy_arn = "arn:aws:iam::aws:policy/service-role/AutoScalingNotificationAccessRole"
}

resource "aws_iam_policy" "lambda_logging102" {
  name = "lambda_logging102"
  path = "/"
  description = "IAM policy for logging from a lambda"

  policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": [
        "logs:CreateLogGroup",
        "logs:CreateLogStream",
        "logs:PutLogEvents"
      ],
      "Resource": "arn:aws:logs:*:*:*",
      "Effect": "Allow"
    }
  ]
}
EOF
}

resource "aws_iam_policy" "lambda_ec2-102" {
  name = "lambda_access_ec2-102"
  path = "/"
  description = "IAM policy for accessing EC2 functions from Lambda"
  policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": [
        "ec2:*"
      ],
      "Resource": "*",
      "Effect": "Allow"
    }
  ]
}
EOF
}

resource "aws_iam_policy" "lambda_sns102" {
  name = "lambda_sns102"
  path = "/"
  description = "IAM policy for SNS"

  policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": [
        "sqs:SendMessage",
        "sqs:GetQueueUrl",
        "sns:Publish"
      ],
      "Resource": "*",
      "Effect": "Allow"
    }
  ]
}
EOF
}

resource "aws_iam_policy" "lambda_ses102" {
  name = "lambda_ses102"
  path = "/"
  description = "IAM policy for SES"

  policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": [
        "ses:*"
      ],
      "Resource": "*",
      "Effect": "Allow"
    }
  ]
}
EOF
}


resource "aws_iam_role_policy_attachment" "lambda_logging102" {
   role = "${aws_iam_role.iam_for_lambda102.name}"
   policy_arn = "${aws_iam_policy.lambda_logging102.arn}"
}

resource "aws_iam_role_policy_attachment" "lambda_sns102" {
   role = "${aws_iam_role.iam_for_lambda102.name}"
   policy_arn = "${aws_iam_policy.lambda_sns102.arn}"
}

resource "aws_iam_role_policy_attachment" "lambda_ses" {
   role = "${aws_iam_role.iam_for_lambda102.name}"
   policy_arn = "${aws_iam_policy.lambda_ses102.arn}"
}

resource "aws_iam_role_policy_attachment" "lambda_ec2" {
   role = "${aws_iam_role.iam_for_lambda102.name}"
   policy_arn = "${aws_iam_policy.lambda_ec2-102.arn}"
}


resource "aws_launch_template" "chuacw102" {
  name_prefix = "chuacw102"
  image_id = "ami-4be75934"
  instance_type = "t2.micro"
}

resource "aws_autoscaling_group" "chuacw102" {
    depends_on = ["aws_iam_role_policy_attachment.lambda_ec2", "aws_iam_role_policy_attachment.lambda_ses",
        "aws_iam_role_policy_attachment.lambda_sns102", "aws_iam_role_policy_attachment.lambda_logging102", "aws_iam_role.iam_for_lambda102",
        "aws_launch_template.chuacw102"]   
    health_check_grace_period = 400
    health_check_type = "ELB"
    availability_zones = ["us-east-1a"]
    desired_capacity = 1
    max_size = 4
    min_size = 1

    initial_lifecycle_hook {
        name = "chuacwlch102"
        default_result = "CONTINUE"
        heartbeat_timeout = 2000
        lifecycle_transition = "autoscaling:EC2_INSTANCE_LAUNCHING"
        notification_target_arn = "${aws_sns_topic.topic102.arn}"
        role_arn = "${aws_iam_role.iam_for_lambda102.arn}"
        notification_metadata = <<EOF
{
  "foo": "bar"
}
EOF
    }

    launch_template = {
        id = "${aws_launch_template.chuacw102.id}"
        version = "$$Latest"
    }
}

// moved into autoscaling group
// resource "aws_autoscaling_lifecycle_hook" "chuacwlch102" {
//   name = "chuacwlch102"
//   autoscaling_group_name = "${aws_autoscaling_group.chuacw102.name}"
//   default_result = "CONTINUE"
//   heartbeat_timeout = 2000
//   lifecycle_transition = "autoscaling:EC2_INSTANCE_LAUNCHING"
//   notification_target_arn = "${aws_sns_topic.topic102.arn}"
//   role_arn = "${aws_iam_role.iam_for_lambda102.arn}"
//   notification_metadata = <<EOF
// {
//   "foo": "bar"
// }
// EOF
// }

Igor Cicimov

unread,

Sep 18, 2018, 12:04:21 AM9/18/18

to Terraform

Well you have:

health_check_type = "ELB"

but where is your ELB? You need to change that to type VPC.

Igor Cicimov

unread,

Sep 18, 2018, 12:08:23 AM9/18/18

to Terraform

Sorry meant EC2 not VPC so correct would be:

health_check_type = "EC2"

On Monday, September 17, 2018 at 2:24:24 PM UTC+10, CheeWee Chua wrote:

CheeWee Chua

unread,

Sep 20, 2018, 11:18:22 PM9/20/18

to Terraform

Thank you for spotting my mistake, Igor!

Very much appreciated!

Reply all

Reply to author

Forward

0 new messages