[COMMIT scylla-cluster-tests master] refactor(aws_region): rename prepare_region.py to aws_region.py

[Commit Bot]

From: Evgeniy Naydanov <evgeniy....@scylladb.com>
Committer: Bentsi <ben...@scylladb.com>
Branch: master

refactor(aws_region): rename prepare_region.py to aws_region.py

- change module name
- rename some properties/methods related to SCT

---
diff --git a/sct.py b/sct.py
--- a/sct.py
+++ b/sct.py
@@ -69,7 +69,7 @@
kill_running_monitoring_stack_services)
from sdcm.cluster import TestConfig
from sdcm.utils.log import setup_stdout_logger
-from sdcm.utils.prepare_region import AwsRegion
+from sdcm.utils.aws_region import AwsRegion
from sdcm.utils.get_username import get_username
from sdcm.send_email import get_running_instances_for_email_report, read_email_data_from_file, build_reporter
from utils.build_system.create_test_release_jobs import JenkinsPipelines
diff --git a/sdcm/sct_runner.py b/sdcm/sct_runner.py
--- a/sdcm/sct_runner.py
+++ b/sdcm/sct_runner.py
@@ -19,7 +19,7 @@
from sdcm.remote import RemoteCmdRunnerBase
from sdcm.utils.aws_utils import ec2_instance_wait_public_ip, ec2_ami_get_root_device_name
from sdcm.utils.get_username import get_username
-from sdcm.utils.prepare_region import AwsRegion
+from sdcm.utils.aws_region import AwsRegion
from sdcm.utils.gce_utils import get_gce_service


@@ -245,7 +245,7 @@ def _create_instance(self, instance_type, base_image, tags_list, instance_name=N
InstanceType=instance_type,
MinCount=1,
MaxCount=1,
- KeyName=aws_region.KEY_PAIR_NAME,
+ KeyName=aws_region.SCT_KEY_PAIR_NAME,
NetworkInterfaces=[{
"DeviceIndex": 0,
"AssociatePublicIpAddress": True,
diff --git a/sdcm/utils/aws_region.py b/sdcm/utils/aws_region.py
--- a/sdcm/utils/aws_region.py
+++ b/sdcm/utils/aws_region.py
@@ -1,3 +1,16 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
import logging
from ipaddress import ip_network
from functools import cached_property
@@ -13,13 +26,13 @@


class AwsRegion:
- VPC_NAME = "SCT-vpc"
- VPC_CIDR = ip_network("10.0.0.0/16")
- SECURITY_GROUP_NAME = "SCT-sg"
- SUBNET_NAME = "SCT-subnet-{availability_zone}"
- INTERNET_GATEWAY_NAME = "SCT-igw"
- ROUTE_TABLE_NAME = "SCT-rt"
- KEY_PAIR_NAME = "scylla-qa-ec2" # TODO: change legacy name to sct-keypair-aws
+ SCT_VPC_NAME = "SCT-vpc"
+ SCT_VPC_CIDR = ip_network("10.0.0.0/16")
+ SCT_SECURITY_GROUP_NAME = "SCT-sg"
+ SCT_SUBNET_NAME = "SCT-subnet-{availability_zone}"
+ SCT_INTERNET_GATEWAY_NAME = "SCT-igw"
+ SCT_ROUTE_TABLE_NAME = "SCT-rt"
+ SCT_KEY_PAIR_NAME = "scylla-qa-ec2" # TODO: change legacy name to sct-keypair-aws

def __init__(self, region_name):
self.region_name = region_name
@@ -28,26 +41,26 @@ def __init__(self, region_name):

@property
def sct_vpc(self) -> EC2ServiceResource.Vpc:
- vpcs = self.client.describe_vpcs(Filters=[{"Name": "tag:Name", "Values": [self.VPC_NAME]}])
+ vpcs = self.client.describe_vpcs(Filters=[{"Name": "tag:Name", "Values": [self.SCT_VPC_NAME]}])
LOGGER.debug("Found VPCs: %s", vpcs)
existing_vpcs = vpcs.get("Vpcs", [])
if len(existing_vpcs) == 0:
return None
assert len(existing_vpcs) == 1, \
- f"More than 1 VPC with {self.VPC_NAME} found in {self.region_name}: {existing_vpcs}"
+ f"More than 1 VPC with {self.SCT_VPC_NAME} found in {self.region_name}: {existing_vpcs}"
return self.resource.Vpc(existing_vpcs[0]["VpcId"]) # pylint: disable=no-member

- def create_vpc(self):
+ def create_sct_vpc(self):
LOGGER.info("Going to create VPC...")
if self.sct_vpc:
- LOGGER.warning("VPC '%s' already exists! Id: '%s'.", self.VPC_NAME, self.sct_vpc.vpc_id)
+ LOGGER.warning("VPC '%s' already exists! Id: '%s'.", self.SCT_VPC_NAME, self.sct_vpc.vpc_id)
return self.sct_vpc.vpc_id
else:
- result = self.client.create_vpc(CidrBlock=str(self.VPC_CIDR), AmazonProvidedIpv6CidrBlock=True)
+ result = self.client.create_vpc(CidrBlock=str(self.SCT_VPC_CIDR), AmazonProvidedIpv6CidrBlock=True)
vpc_id = result["Vpc"]["VpcId"]
vpc = self.resource.Vpc(vpc_id) # pylint: disable=no-member
- vpc.create_tags(Tags=[{"Key": "Name", "Value": self.VPC_NAME}])
- LOGGER.info("'%s' with id '%s' created. Waiting until it becomes available...", self.VPC_NAME, vpc_id)
+ vpc.create_tags(Tags=[{"Key": "Name", "Value": self.SCT_VPC_NAME}])
+ LOGGER.info("'%s' with id '%s' created. Waiting until it becomes available...", self.SCT_VPC_NAME, vpc_id)
vpc.wait_until_available()
return vpc_id

@@ -61,7 +74,7 @@ def vpc_ipv6_cidr(self):
return ip_network(self.sct_vpc.ipv6_cidr_block_association_set[0]["Ipv6CidrBlock"])

def az_subnet_name(self, region_az):
- return self.SUBNET_NAME.format(availability_zone=region_az)
+ return self.SCT_SUBNET_NAME.format(availability_zone=region_az)

def sct_subnet(self, region_az) -> EC2ServiceResource.Subnet:
subnet_name = self.az_subnet_name(region_az)
@@ -74,7 +87,7 @@ def sct_subnet(self, region_az) -> EC2ServiceResource.Subnet:
f"More than 1 Subnet with {subnet_name} found in {self.region_name}: {existing_subnets}!"
return self.resource.Subnet(existing_subnets[0]["SubnetId"]) # pylint: disable=no-member

- def create_subnet(self, region_az, ipv4_cidr, ipv6_cidr):
+ def create_sct_subnet(self, region_az, ipv4_cidr, ipv6_cidr):
LOGGER.info("Creating subnet for %s...", region_az)
subnet_name = self.az_subnet_name(region_az)
if self.sct_subnet(region_az):
@@ -99,65 +112,65 @@ def create_subnet(self, region_az, ipv4_cidr, ipv6_cidr):
)
LOGGER.info("'%s' with id '%s' created.", subnet_name, subnet_id)

- def create_subnets(self):
+ def create_sct_subnets(self):
num_subnets = len(self.availability_zones)
- ipv4_cidrs = list(self.VPC_CIDR.subnets(6))[:num_subnets]
+ ipv4_cidrs = list(self.SCT_VPC_CIDR.subnets(6))[:num_subnets]
ipv6_cidrs = list(self.vpc_ipv6_cidr.subnets(8))[:num_subnets]
for i, az_name in enumerate(self.availability_zones):
- self.create_subnet(region_az=az_name, ipv4_cidr=ipv4_cidrs[i], ipv6_cidr=ipv6_cidrs[i])
+ self.create_sct_subnet(region_az=az_name, ipv4_cidr=ipv4_cidrs[i], ipv6_cidr=ipv6_cidrs[i])

@property
def sct_internet_gateway(self) -> EC2ServiceResource.InternetGateway:
igws = self.client.describe_internet_gateways(Filters=[{"Name": "tag:Name",
- "Values": [self.INTERNET_GATEWAY_NAME]}])
+ "Values": [self.SCT_INTERNET_GATEWAY_NAME]}])
LOGGER.debug("Found Internet gateways: %s", igws)
existing_igws = igws.get("InternetGateways", [])
if len(existing_igws) == 0:
return None
assert len(existing_igws) == 1, \
- f"More than 1 Internet Gateway with {self.INTERNET_GATEWAY_NAME} found " \
+ f"More than 1 Internet Gateway with {self.SCT_INTERNET_GATEWAY_NAME} found " \
f"in {self.region_name}: {existing_igws}!"
return self.resource.InternetGateway(existing_igws[0]["InternetGatewayId"]) # pylint: disable=no-member

- def create_internet_gateway(self):
+ def create_sct_internet_gateway(self):
LOGGER.info("Creating Internet Gateway..")
if self.sct_internet_gateway:
LOGGER.warning("Internet Gateway '%s' already exists! Id: '%s'.",
- self.INTERNET_GATEWAY_NAME, self.sct_internet_gateway.internet_gateway_id)
+ self.SCT_INTERNET_GATEWAY_NAME, self.sct_internet_gateway.internet_gateway_id)
else:
result = self.client.create_internet_gateway()
igw_id = result["InternetGateway"]["InternetGatewayId"]
igw = self.resource.InternetGateway(igw_id) # pylint: disable=no-member
- igw.create_tags(Tags=[{"Key": "Name", "Value": self.INTERNET_GATEWAY_NAME}])
+ igw.create_tags(Tags=[{"Key": "Name", "Value": self.SCT_INTERNET_GATEWAY_NAME}])
LOGGER.info("'%s' with id '%s' created. Attaching to '%s'",
- self.INTERNET_GATEWAY_NAME, igw_id, self.sct_vpc.vpc_id)
+ self.SCT_INTERNET_GATEWAY_NAME, igw_id, self.sct_vpc.vpc_id)
igw.attach_to_vpc(VpcId=self.sct_vpc.vpc_id)

@cached_property
def sct_route_table(self) -> EC2ServiceResource.RouteTable:
route_tables = self.client.describe_route_tables(Filters=[{"Name": "tag:Name",
- "Values": [self.ROUTE_TABLE_NAME]}])
+ "Values": [self.SCT_ROUTE_TABLE_NAME]}])
LOGGER.debug("Found Route Tables: %s", route_tables)
existing_rts = route_tables.get("RouteTables", [])
if len(existing_rts) == 0:
return None
assert len(existing_rts) == 1, \
- f"More than 1 Route Table with {self.ROUTE_TABLE_NAME} found " \
+ f"More than 1 Route Table with {self.SCT_ROUTE_TABLE_NAME} found " \
f"in {self.region_name}: {existing_rts}!"
return self.resource.RouteTable(existing_rts[0]["RouteTableId"]) # pylint: disable=no-member

- def configure_route_table(self):
+ def configure_sct_route_table(self):
# add route to Internet: 0.0.0.0/0 -> igw
LOGGER.info("Configuring main Route Table...")
if self.sct_route_table:
LOGGER.warning("Route Table '%s' already exists! Id: '%s'.",
- self.ROUTE_TABLE_NAME, self.sct_route_table.route_table_id)
+ self.SCT_ROUTE_TABLE_NAME, self.sct_route_table.route_table_id)
else:
route_tables = list(self.sct_vpc.route_tables.all())
- assert len(route_tables) == 1, f"Only one main route table should exist for {self.VPC_NAME}. " \
+ assert len(route_tables) == 1, f"Only one main route table should exist for {self.SCT_VPC_NAME}. " \
f"Found {len(route_tables)}!"
route_table: EC2ServiceResource.RouteTable = route_tables[0]
- route_table.create_tags(Tags=[{"Key": "Name", "Value": self.ROUTE_TABLE_NAME}])
+ route_table.create_tags(Tags=[{"Key": "Name", "Value": self.SCT_ROUTE_TABLE_NAME}])
LOGGER.info("Setting routing of all outbound traffic via Internet Gateway...")
route_table.create_route(DestinationCidrBlock="0.0.0.0/0",
GatewayId=self.sct_internet_gateway.internet_gateway_id)
@@ -172,17 +185,17 @@ def configure_route_table(self):
@property
def sct_security_group(self) -> EC2ServiceResource.SecurityGroup:
security_groups = self.client.describe_security_groups(Filters=[{"Name": "tag:Name",
- "Values": [self.SECURITY_GROUP_NAME]}])
+ "Values": [self.SCT_SECURITY_GROUP_NAME]}])
LOGGER.debug("Found Security Groups: %s", security_groups)
existing_sgs = security_groups.get("SecurityGroups", [])
if len(existing_sgs) == 0:
return None
assert len(existing_sgs) == 1, \
- f"More than 1 Security group with {self.SECURITY_GROUP_NAME} found " \
+ f"More than 1 Security group with {self.SCT_SECURITY_GROUP_NAME} found " \
f"in {self.region_name}: {existing_sgs}!"
return self.resource.SecurityGroup(existing_sgs[0]["GroupId"]) # pylint: disable=no-member

- def create_security_group(self):
+ def create_sct_security_group(self):
"""

Custom TCP TCP 9093 0.0.0.0/0 Allow alert manager for all
@@ -192,15 +205,15 @@ def create_security_group(self):
LOGGER.info("Creating Security Group...")
if self.sct_security_group:
LOGGER.warning("Security Group '%s' already exists! Id: '%s'.",
- self.SECURITY_GROUP_NAME, self.sct_security_group.group_id)
+ self.SCT_SECURITY_GROUP_NAME, self.sct_security_group.group_id)
else:
result = self.client.create_security_group(Description='Security group that is used by SCT',
- GroupName=self.SECURITY_GROUP_NAME,
+ GroupName=self.SCT_SECURITY_GROUP_NAME,
VpcId=self.sct_vpc.vpc_id)
sg_id = result["GroupId"]
security_group = self.resource.SecurityGroup(sg_id) # pylint: disable=no-member
- security_group.create_tags(Tags=[{"Key": "Name", "Value": self.SECURITY_GROUP_NAME}])
- LOGGER.info("'%s' with id '%s' created. ", self.SECURITY_GROUP_NAME, self.sct_security_group.group_id)
+ security_group.create_tags(Tags=[{"Key": "Name", "Value": self.SCT_SECURITY_GROUP_NAME}])
+ LOGGER.info("'%s' with id '%s' created. ", self.SCT_SECURITY_GROUP_NAME, self.sct_security_group.group_id)
LOGGER.info("Creating common ingress rules...")
security_group.authorize_ingress(
IpPermissions=[
@@ -367,41 +380,35 @@ def create_security_group(self):
@property
def sct_keypair(self):
try:
- key_pairs = self.client.describe_key_pairs(KeyNames=[self.KEY_PAIR_NAME])
+ key_pairs = self.client.describe_key_pairs(KeyNames=[self.SCT_KEY_PAIR_NAME])
except botocore.exceptions.ClientError as ex:
if "InvalidKeyPair.NotFound" in str(ex):
return None
- else:
- raise
+ raise
LOGGER.debug("Found key pairs: %s", key_pairs)
existing_key_pairs = key_pairs.get("KeyPairs", [])
assert len(existing_key_pairs) == 1, \
- f"More than 1 Key Pair with {self.KEY_PAIR_NAME} found " \
+ f"More than 1 Key Pair with {self.SCT_KEY_PAIR_NAME} found " \
f"in {self.region_name}: {existing_key_pairs}!"
return self.resource.KeyPair(existing_key_pairs[0]["KeyName"]) # pylint: disable=no-member

- def create_key_pair(self):
+ def create_sct_key_pair(self):
LOGGER.info("Creating SCT Key Pair...")
if self.sct_keypair:
LOGGER.warning("SCT Key Pair already exists in '%s'!", self.region_name)
else:
ks = KeyStore()
sct_key_pair = ks.get_ec2_ssh_key_pair()
- self.resource.import_key_pair(KeyName=self.KEY_PAIR_NAME, # pylint: disable=no-member
+ self.resource.import_key_pair(KeyName=self.SCT_KEY_PAIR_NAME, # pylint: disable=no-member
PublicKeyMaterial=sct_key_pair.public_key)
LOGGER.info("SCT Key Pair created.")

def configure(self):
LOGGER.info("Configuring '%s' region...", self.region_name)
- self.create_vpc()
- self.create_subnets()
- self.create_internet_gateway()
- self.configure_route_table()
- self.create_security_group()
- self.create_key_pair()
+ self.create_sct_vpc()
+ self.create_sct_subnets()
+ self.create_sct_internet_gateway()
+ self.configure_sct_route_table()
+ self.create_sct_security_group()
+ self.create_sct_key_pair()
LOGGER.info("Region configured successfully.")
-
-
-if __name__ == "__main__":
- AWS_REGION = AwsRegion(region_name="eu-west-2")
- AWS_REGION.configure()
diff --git a/sdcm/utils/aws_utils.py b/sdcm/utils/aws_utils.py
--- a/sdcm/utils/aws_utils.py
+++ b/sdcm/utils/aws_utils.py
@@ -1,14 +1,26 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
import time
import logging
from functools import cached_property
from typing import List, Dict

import boto3
-
from botocore.exceptions import ClientError

from sdcm.utils.decorators import retrying
-from sdcm.utils.prepare_region import AwsRegion
+from sdcm.utils.aws_region import AwsRegion
from sdcm.wait import wait_for