[COMMIT scylla-cluster-tests master] feature(sdcm): implement ScyllaYaml

68 views

Skip to first unread message

Commit Bot

<bot@cloudius-systems.com>

unread,

Sep 28, 2021, 8:11:01 AM9/28/21

to scylladb-dev@googlegroups.com, Dmitry Kropachev

From: Dmitry Kropachev <dmitry.k...@gmail.com>
Committer: Bentsi <ben...@scylladb.com>
Branch: master

feature(sdcm): implement ScyllaYaml

Create ScyllaYaml and it's builders that
would build parameters out of SctConfiguration

---
diff --git a/docker/env/version b/docker/env/version
--- a/docker/env/version
+++ b/docker/env/version
@@ -1 +1 @@
-0.95
+0.96
diff --git a/requirements.in b/requirements.in
--- a/requirements.in
+++ b/requirements.in
@@ -46,3 +46,4 @@ azure-mgmt-network==19.0.0
azure-mgmt-resource==20.0.0
azure-mgmt-subscription==1.0.0
azure-mgmt-resourcegraph==8.0.0
+pydantic==1.8.2
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,18 +2,18 @@
# This file is autogenerated by pip-compile
# To update, run:
#
-# pip-compile --allow-unsafe --generate-hashes requirements.in
+# pip-compile --allow-unsafe --generate-hashes /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
#
aexpect==1.2.0 \
--hash=sha256:69a45a4e8498f7041b9e0e203edf392dc7ba2674f0f72b2221c574e4694a34d8 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
anyconfig==0.9.8 \
--hash=sha256:9de54c3252240b3c562e25ef070e883c266b7daad8c8bc4d5d604855edc7d739 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
apache-libcloud==2.6.0 \
--hash=sha256:201751f738109f25d58dcdfb5804e17216e0dc8f68b522e9e26ac16e0b9ff2ea \
--hash=sha256:40215db1bd489d17dc1abfdb289d7f035313c7297b6a7462c79d8287cbbeae91 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
astroid==2.5.6 \
--hash=sha256:4db03ab5fc3340cf619dbc25e42c2cc3755154ce6009469766d7143d1fc2ee4e \
--hash=sha256:8a398dfce302c13f14bab13e2b14fe385d32b73f4e4853b9bdfb64598baa1975 \
@@ -28,11 +28,11 @@ attrs==21.2.0 \
# via pytest
autopep8==1.5.3 \
--hash=sha256:60fd8c4341bab59963dafd5d2a566e94f547e660b9b396f772afe67d8481dbf0 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
awscli==1.19.3 \
--hash=sha256:062030352fe47b6d1f91df61d26183c4077c69cd5b9be9801273a39ab2d4f415 \
--hash=sha256:3f34cbaffe382cc66e6841b477b1836b762174cd9ea58581dce59323ff47b5c4 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
azure-common==1.1.27 \
--hash=sha256:426673962740dbe9aab052a4b52df39c07767decd3f25fdc87c9d4c566a04934 \
--hash=sha256:9f3f5d991023acbd93050cf53c4e863c6973ded7e236c69e99c8ff5c7bad41ef \
@@ -44,31 +44,31 @@ azure-core==1.18.0 \
azure-identity==1.6.1 \
--hash=sha256:69035c81f280fac5fa9c55f87be3a359b264853727486e3568818bb43988080e \
--hash=sha256:6f4f8c1ba2887cf3e1663be324fde278fce52d781123ad95c07d2c9d9cdeef5a \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
azure-mgmt-compute==23.0.0 \
--hash=sha256:1eb26b965ba4049ddcf10d4f25818725fc03c491c3be76537d0d74ceb1146b04 \
--hash=sha256:3efc8084744d1f275a6eb2ed8c920da34f54b0c8c97acb217ea34c9f6821824e \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
azure-mgmt-core==1.3.0 \
--hash=sha256:3ffb7352b39e5495dccc2d2b47254f4d82747aff4735e8bf3267c335b0c9bb40 \
--hash=sha256:7b7fa952aeb9d3eaa13eff905880f3d3b62200f7be7a8ba5a50c8b2e7295322a \
# via azure-mgmt-compute, azure-mgmt-network, azure-mgmt-resource, azure-mgmt-resourcegraph, azure-mgmt-subscription
azure-mgmt-network==19.0.0 \
--hash=sha256:4585c5eedcb8783c2840feef1d66843849f39835d7e3f2ec8a5af834fd7b949e \
--hash=sha256:5e39a26ae81fa58c13c02029700f8c7b22c3fd832a294c543e3156a91b9459e8 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
azure-mgmt-resource==20.0.0 \
--hash=sha256:0ebad05a946e27c17d5a86ea5a68b2d9a0142257656ae8211dee9fc191c849b7 \
--hash=sha256:622dca4484be64f9f5ce335d327dffabf3e71e14e8a3f4a1051dc85a5c3ebbca \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
azure-mgmt-resourcegraph==8.0.0 \
--hash=sha256:0cf55f7ea82dc03e69d0fae0f1606e09b08b80b6ae23bd597d8b62b1ed938ace \
--hash=sha256:d25f01dae3897780fb3ddca16d1625b6347c32f1b581c767fba5ef3b24443f11 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
azure-mgmt-subscription==1.0.0 \
--hash=sha256:22f606f298419f466a8149811fc762686c93da00a7dc15d3b7cdbf22b96cf5db \
--hash=sha256:438e1373851b8616ef4e07ae3ab50f7654e830a0cac83ff3c67d0124825f0f06 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
backports.entry-points-selectable==1.1.0 \
--hash=sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a \
--hash=sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc \
@@ -85,11 +85,11 @@ bcrypt==3.2.0 \
boto3-stubs[dynamodb,ec2,pricing,s3]==1.17.3.0 \
--hash=sha256:54cd69236abe4742f90cb8b45ed76cbc8ba7885cf020d6d48b91fd3f554ba6ee \
--hash=sha256:c26c98203f94eb463ab0f13372e092ba605218002643a0915cf31cb53fcd957f \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
boto3==1.17.3 \
--hash=sha256:92041aa7589c886020cabd80eb58b89ace2f0094571792fccae24b9a8b3b97d7 \
--hash=sha256:9f132c34e20110dea019293c89cede49b0a56be615b3e1debf98390ed9f1f7b9 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
botocore==1.20.3 \
--hash=sha256:1dae84c68b109f596f58cc2e9fa87704ccd40dcbc12144a89205f85efa7f9135 \
--hash=sha256:a0fdded1c9636899ab273f50bf123f79b91439a8c282b5face8b5f4a48b493cb \
@@ -127,7 +127,7 @@ cassandra-driver==3.22.0 \
--hash=sha256:f3fef1fe2da69f8ed126755f3f26c85fce84ed3bc59f0ea41fde2125f7569799 \
--hash=sha256:f59267674c6b1c5f7303b08ff9dac0d3f218e3737a848fa105718e6cbe7d4bf5 \
--hash=sha256:f74eb7570c933ab4bd3d8f9080497eced9f57d24a79796c352e78a3af1a0a5f6 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
certifi==2021.5.30 \
--hash=sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee \
--hash=sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8 \
@@ -189,11 +189,11 @@ chardet==4.0.0 \
# via mbstrdecoder, requests
click-completion==0.5.0 \
--hash=sha256:7600261cda0954b9794bd7afdaab35201be50a8c5404eaa4c0deb20086866c4c \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
click==7.0 \
--hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
--hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
- # via -r requirements.in, click-completion, geomet, pip-tools
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in, click-completion, geomet, pip-tools
colorama==0.4.3 \
--hash=sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff \
--hash=sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1 \
@@ -204,8 +204,10 @@ cryptography==3.4.8 \
--hash=sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7 \
--hash=sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085 \
--hash=sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc \
+ --hash=sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d \
--hash=sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a \
--hash=sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498 \
+ --hash=sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89 \
--hash=sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9 \
--hash=sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c \
--hash=sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7 \
@@ -217,46 +219,46 @@ cryptography==3.4.8 \
--hash=sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06 \
--hash=sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7 \
# via azure-identity, msal, paramiko, pyjwt
-dataproperty==0.52.0 \
- --hash=sha256:8fda054fcc80f01e6c1c91e4853acd6982c99fdc91fb96f536d073c6ddaa2a5a \
- --hash=sha256:b16885d394c2fa4a024f2e4318f9e387d909fd7e35d4cf6d1814c57d812fe21d \
+dataproperty==0.53.0 \
+ --hash=sha256:9589b5c5b400a77e1aeb63d89d755043bbe00043219336963c93db5df49cbaf7 \
+ --hash=sha256:decf41325e8052fa6762867e56907e632b3276d84cbd859c4650034b342456e2 \
# via simplesqlite, tabledata, tcconfig
-distlib==0.3.2 \
- --hash=sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736 \
- --hash=sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c \
+distlib==0.3.3 \
+ --hash=sha256:c8b54e8454e5bf6237cc84c20e8264c3e991e824ef27e8f1e81049867d861e31 \
+ --hash=sha256:d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05 \
# via virtualenv
docker==4.2.0 \
--hash=sha256:1c2ddb7a047b2599d1faec00889561316c674f7099427b9c51e8cb804114b553 \
--hash=sha256:ddae66620ab5f4bce769f64bcd7934f880c8abe6aa50986298db56735d0f722e \
- # via -r requirements.in, tcconfig
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in, tcconfig
docutils==0.15.2 \
--hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
--hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
--hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \
# via awscli
-elasticsearch==7.14.1 \
- --hash=sha256:1a9f146b7126a7e0621085f1825b6b2d091693a714d3b16c208749762e79c2bc \
- --hash=sha256:f928898fe06869516f2603f9a96a6f166c06888233806b31ac6568bac0266501 \
- # via -r requirements.in
+elasticsearch==7.15.0 \
+ --hash=sha256:6c47eb68f643f3d7d6a6d971e571448b900b9470dc481421c9af9d28a1303911 \
+ --hash=sha256:bf1d210d82c78fbd3ae5cbedd756716d732433cc97184966688dcd1c75df4619 \
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
fabric==2.5.0 \
--hash=sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389 \
--hash=sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6 \
- # via -r requirements.in
-filelock==3.0.12 \
- --hash=sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59 \
- --hash=sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836 \
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
+filelock==3.1.0 \
+ --hash=sha256:78925788ce8c8945fac28a68c1d05cf33a6a6c4fba14fe02835122c53268ceef \
+ --hash=sha256:d9e9c7d8191e915339843c81c90d3e44f7c84e5fb03bdc6b1b4d019025cf953b \
# via virtualenv
geomet==0.1.2 \
--hash=sha256:cef6c73cfc0c4ea3961e16a6979dce75ef0298f0023cbd482855134dcdf7c010 \
# via cassandra-driver
-google-api-core==1.31.2 \
- --hash=sha256:384459a0dc98c1c8cd90b28dc5800b8705e0275a673a7144a513ae80fc77950b \
- --hash=sha256:8500aded318fdb235130bf183c726a05a9cb7c4b09c266bd5119b86cdb8a4d10 \
+google-api-core==1.31.3 \
+ --hash=sha256:4b7ad965865aef22afa4aded3318b8fa09b20bcc7e8dbb639a3753cf60af08ea \
+ --hash=sha256:f52c708ab9fd958862dea9ac94d9db1a065608073fe583c3b9c18537b177f59a \
# via google-api-python-client
google-api-python-client==2.1.0 \
--hash=sha256:921fe10cdff22d1f5b8af7473f9a298efb991fb6ea67dadd6c37fbecfb0575f4 \
--hash=sha256:f9ac377efe69571aea1acc9e15760d4204aca23c4464eb63f963ae4defc95d97 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
google-auth-httplib2==0.1.0 \
--hash=sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10 \
--hash=sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac \
@@ -277,9 +279,9 @@ humanreadable==0.1.0 \
--hash=sha256:80f10a1575ebb140d9345a347f981dc6faa70d090885490d77f63361290b4ff7 \
--hash=sha256:c925e8d805d7a29ff089d11be92a5b80f4545e9e44b20e01d23a65b278a9b573 \
# via tcconfig
-identify==2.2.14 \
- --hash=sha256:113a76a6ba614d2a3dd408b3504446bcfac0370da5995aa6a17fd7c6dffde02d \
- --hash=sha256:32f465f3c48083f345ad29a9df8419a4ce0674bf4a8c3245191d65c83634bdbf \
+identify==2.2.15 \
+ --hash=sha256:528a88021749035d5a39fe2ba67c0642b8341aaf71889da0e1ed669a429b87f0 \
+ --hash=sha256:de83a84d774921669774a2000bf87ebba46b4d1c04775f4a5d37deff0cf39f73 \
# via pre-commit
idna==2.10 \
--hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
@@ -293,7 +295,7 @@ invoke==1.3.1 \
--hash=sha256:5a8558521dc5621b2483a1d90944567e2e104e09dda7be6ae4079eb3247f4a3b \
--hash=sha256:7e7659e290e375011adf828a491f07e341852b2126e05de14459408b66199915 \
--hash=sha256:dae041ff458e1ef05448aae3b76e8c2a176c4b7c6a9d5e8ce880f16251803661 \
- # via -r requirements.in, fabric
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in, fabric
isodate==0.6.0 \
--hash=sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8 \
--hash=sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81 \
@@ -305,15 +307,15 @@ isort==5.9.3 \
jinja2==2.11.3 \
--hash=sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419 \
--hash=sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6 \
- # via -r requirements.in, click-completion
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in, click-completion
jmespath==0.10.0 \
--hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
--hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \
# via boto3, botocore
kubernetes==12.0.1 \
--hash=sha256:23c85d8571df8f56e773f1a413bc081537536dc47e2b5e8dc2e6262edb2c57ca \
--hash=sha256:ec52ea01d52e2ec3da255992f7e859f3a76f2bdb51cf65ba8cd71dfc309d8daa \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
lazy-object-proxy==1.6.0 \
--hash=sha256:17e0967ba374fc24141738c69736da90e94419338fd4c7c7bef01ee26b339653 \
--hash=sha256:1fee665d2638491f4d6e55bd483e15ef21f6c8c2095f235fef72601021e64f61 \
@@ -341,7 +343,7 @@ lazy-object-proxy==1.6.0 \
ldap3==2.7 \
--hash=sha256:17f04298b70bf7ecaa5db8a7d8622b5a962ef7fc2b245b2eea705ac1c24338c0 \
--hash=sha256:81df4ac8b6df10fb1f05b17c18d0cb8c4c344d5a03083c382824960ed959cf5b \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
loguru==0.5.3 \
--hash=sha256:b28e72ac7a98be3d28ad28570299a393dfcd32e5e3f6a353dec94675767b6319 \
--hash=sha256:f8087ac396b5ee5f67c963b495d615ebbceac2796379599820e324419d53667c \
@@ -402,17 +404,17 @@ markupsafe==2.0.1 \
--hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \
--hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 \
# via jinja2
-mbstrdecoder==1.0.1 \
- --hash=sha256:56b967800d6554815a6b18f98f915e84912852eff21d94d5e036fe124bc27f53 \
- --hash=sha256:f895e1fb97496855ab5e43de99588787169c01f782c71625142d1a62729e9f9d \
+mbstrdecoder==1.1.0 \
+ --hash=sha256:153443c34b5e0c9b2263a1f8480d33156190b3e03e1e89cccd9239581aaef0ed \
+ --hash=sha256:f4dfd549e424ad8dfc985e6af8b55cb4ec0c208782f610d57439fe6a9a44c244 \
# via dataproperty, simplesqlite, sqliteschema, subprocrunner, typepy
mccabe==0.6.1 \
--hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
--hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
# via pylint
-more-itertools==8.9.0 \
- --hash=sha256:70401259e46e216056367a0a6034ee3d3f95e0bf59d3aa6a4eb77837171ed996 \
- --hash=sha256:8c746e0d09871661520da4f1241ba6b908dc903839733c8203b552cffaf173bd \
+more-itertools==8.10.0 \
+ --hash=sha256:1debcabeb1df793814859d64a81ad7cb10504c24349368ccf214c664c474f41f \
+ --hash=sha256:56ddac45541718ba332db05f464bebfb0768110111affd27f66e0051f276fa43 \
# via pytest
msal-extensions==0.3.0 \
--hash=sha256:5523dfa15da88297e90d2e73486c8ef875a17f61ea7b7e2953a300432c2e7861 \
@@ -474,7 +476,7 @@ mysql-connector-python==8.0.20 \
--hash=sha256:eec4c549d96d9678fe9df663226fc175562dcd7187af892bfe4711c9a0d99228 \
--hash=sha256:f23c8ddc1550def024ea708d2aa7c999f301cd45ff17ca7bf9a762b48f0d4513 \
--hash=sha256:ffe0bdd9b22a987028ca4e70df770edef987d54899ac5029c364d9c3c04e9e9d \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
nodeenv==1.6.0 \
--hash=sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b \
--hash=sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7 \
@@ -490,11 +492,11 @@ packaging==21.0 \
parameterized==0.7.4 \
--hash=sha256:190f8cc7230eee0b56b30d7f074fd4d165f7c45e6077582d0813c8557e738490 \
--hash=sha256:59ab908e31c01505a987a2be78854e19cb1630c047bbab7848169c371d614d56 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
paramiko==2.7.2 \
--hash=sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898 \
--hash=sha256:7f36f4ba2c0d81d219f4595e35f70d56cc94f9ac40a6acdf51d6ca210ce65035 \
- # via -r requirements.in, fabric
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in, fabric
path.py==12.5.0 \
--hash=sha256:8d885e8b2497aed005703d94e0fd97943401f035e42a136810308bff034529a8 \
--hash=sha256:a43e82eb2c344c3fd0b9d6352f6b856f40b8b7d3d65cc05978b42c3715668496 \
@@ -503,9 +505,9 @@ path==16.2.0 \
--hash=sha256:2de925e8d421f93bcea80d511b81accfb6a7e6b249afa4a5559557b0cf817097 \
--hash=sha256:340054c5bb459fc9fd40e7eb6768c5989f3e599d18224238465b5333bc8faa7d \
# via path.py
-pathvalidate==2.4.1 \
- --hash=sha256:3c9bd94c7ec23e9cfb211ffbe356ae75f979d6c099a2c745ee9490f524f32468 \
- --hash=sha256:f5dde7efeeb4262784c5e1331e02752d07c1ec3ee5ea42683fe211155652b808 \
+pathvalidate==2.5.0 \
+ --hash=sha256:119ba36be7e9a405d704c7b7aea4b871c757c53c9adc0ed64f40be1ed8da2781 \
+ --hash=sha256:e5b2747ad557363e8f4124f0553d68878b12ecabd77bcca7e7312d5346d20262 \
# via simplesqlite
pbr==5.6.0 \
--hash=sha256:42df03e7797b796625b1029c0400279c7c34fd7df24a7d7818a1abb5b38710dd \
@@ -514,10 +516,10 @@ pbr==5.6.0 \
pip-tools==5.4.0 \
--hash=sha256:a4d3990df2d65961af8b41dacc242e600fdc8a65a2e155ed3d2fc18a5c209f20 \
--hash=sha256:b73f76fe6464b95e41d595a9c0302c55a786dbc54b63ae776c540c04e31914fb \
- # via -r requirements.in
-platformdirs==2.3.0 \
- --hash=sha256:15b056538719b1c94bdaccb29e5f81879c7f7f0f4a153f46086d155dffcd4f0f \
- --hash=sha256:8003ac87717ae2c7ee1ea5a84a1a61e87f3fbd16eb5aadba194ea30a9019f648 \
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
+platformdirs==2.4.0 \
+ --hash=sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2 \
+ --hash=sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d \
# via virtualenv
pluggy==0.13.1 \
--hash=sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0 \
@@ -530,11 +532,11 @@ portalocker==1.7.1 \
pre-commit==2.13.0 \
--hash=sha256:764972c60693dc668ba8e86eb29654ec3144501310f7198742a767bec385a378 \
--hash=sha256:b679d0fddd5b9d6d98783ae5f10fd0c4c59954f375b70a58cbe1ce9bcf9809a4 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
prometheus_client==0.8.0 \
--hash=sha256:983c7ac4b47478720db338f1491ef67a100b474e3bc7dafcbaefb7d0b8f9b01c \
--hash=sha256:c6e6b706833a6bd1fd51711299edee907857be10ece535126a158f911ee80915 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
protobuf==3.17.3 \
--hash=sha256:13ee7be3c2d9a5d2b42a1030976f760f28755fcf5863c55b1460fd205e6cd637 \
--hash=sha256:145ce0af55c4259ca74993ddab3479c78af064002ec8227beb3d944405123c71 \
@@ -566,7 +568,7 @@ protobuf==3.17.3 \
# via google-api-core, googleapis-common-protos, mysql-connector-python
ptable==0.9.2 \
--hash=sha256:aa7fc151cb40f2dabcd2275ba6f7fd0ff8577a86be3365cd3fb297cbe09cc292 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
py==1.10.0 \
--hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \
--hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a \
@@ -623,15 +625,39 @@ pycryptodome==3.9.8 \
--hash=sha256:f521178e5a991ffd04182ed08f552daca1affcb826aeda0e1945cd989a9d4345 \
--hash=sha256:f78a68c2c820e4731e510a2df3eef0322f24fde1781ced970bf497b6c7d92982 \
--hash=sha256:fbe65d5cfe04ff2f7684160d50f5118bdefb01e3af4718eeb618bfed40f19d94 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
+pydantic==1.8.2 \
+ --hash=sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd \
+ --hash=sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739 \
+ --hash=sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f \
+ --hash=sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840 \
+ --hash=sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23 \
+ --hash=sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287 \
+ --hash=sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62 \
+ --hash=sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b \
+ --hash=sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb \
+ --hash=sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820 \
+ --hash=sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3 \
+ --hash=sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b \
+ --hash=sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e \
+ --hash=sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3 \
+ --hash=sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316 \
+ --hash=sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b \
+ --hash=sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4 \
+ --hash=sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20 \
+ --hash=sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e \
+ --hash=sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505 \
+ --hash=sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1 \
+ --hash=sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833 \
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
pyjwt[crypto]==2.1.0 \
--hash=sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1 \
--hash=sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130 \
# via msal
pylint==2.8.3 \
--hash=sha256:0a049c5d47b629d9070c3932d13bff482b12119b6a241a93bc460b0be16953c8 \
--hash=sha256:792b38ff30903884e4a9eab814ee3523731abd3c463f3ba48d7b627e87013484 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
pynacl==1.4.0 \
--hash=sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4 \
--hash=sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4 \
@@ -656,46 +682,46 @@ pyparsing==2.4.7 \
--hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \
--hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
# via httplib2, packaging, tcconfig
-pyroute2.core==0.6.4 \
- --hash=sha256:453dee5241e32994fb0ccad4c1a54fc98353fec35690653ccf95617b54e3a5cd \
+pyroute2.core==0.6.5 \
+ --hash=sha256:266d740eae40fa64dd04541f007d34db696cecf3154cba5be30fa75a67d43852 \
# via pyroute2, pyroute2.ethtool, pyroute2.ipdb, pyroute2.ipset, pyroute2.ndb, pyroute2.nftables, pyroute2.nslink
-pyroute2.ethtool==0.6.4 \
- --hash=sha256:fa848c8a04eba0a4ce454c0ab5a5be4f10d6dc6fd19dc92a93bae561ade89d13 \
+pyroute2.ethtool==0.6.5 \
+ --hash=sha256:caf8014b676521135c4760d72cf5aeef6abbc7fa275210f7e95333073cc7715a \
# via pyroute2
-pyroute2.ipdb==0.6.4 \
- --hash=sha256:e53c18f293890fab72d4f6cdebf5c7c6e14c9452f0d04b13184ce134e0c19d64 \
+pyroute2.ipdb==0.6.5 \
+ --hash=sha256:f2028fd1013d8af888150d033eecf7537657a4be37e0cccea9800809961eb577 \
# via pyroute2
-pyroute2.ipset==0.6.4 \
- --hash=sha256:57a694198bf83c68711e812380dbaaa116dde9fb6a8ab3bfa1f3434040024f2f \
+pyroute2.ipset==0.6.5 \
+ --hash=sha256:ae527c0f999748230a1f688d9a2b7c257b2df6d74369f44eca530d007793b79d \
# via pyroute2
-pyroute2.ndb==0.6.4 \
- --hash=sha256:7693e6327b973034b92d9451b215342e16d86f477d1b4a626fd01e6eacf07760 \
+pyroute2.ndb==0.6.5 \
+ --hash=sha256:a4d309584e9ef6c7842af4f832b48fc5344ab225e70e384baed1b7fe2b94d9f8 \
# via pyroute2
-pyroute2.nftables==0.6.4 \
- --hash=sha256:c176ffeee5815418dc462904d420c2eebb9cb1f9361c5d9925d34583e0494856 \
+pyroute2.nftables==0.6.5 \
+ --hash=sha256:b1455a63a3efc050d108d550d1cafd011edded6e894d99e7bdfa02d592bf6f16 \
# via pyroute2
-pyroute2.nslink==0.6.4 \
- --hash=sha256:33e68a9431cb5fa0663d508f3670ee4e20d5fabd8d0b0a6144c7f40276dee447 \
+pyroute2.nslink==0.6.5 \
+ --hash=sha256:292e6c2832a73544c1c6d5ba727f7117aa847e55f88d78e94b7d4607b299999e \
# via pyroute2
-pyroute2==0.6.4 \
- --hash=sha256:560b48a751b1150056ba553c89a31d563cc18ae2675b3793666adcaeb4fabfda \
+pyroute2==0.6.5 \
+ --hash=sha256:d0995c8aeb96c0e4eed4d62e97c9d795000a1a33b747d6dc54326665857cf11c \
# via tcconfig
pytest-random-order==1.0.4 \
--hash=sha256:6b2159342a4c8c10855bc4fc6d65ee890fc614cb2b4ff688979b008a82a0ff52 \
--hash=sha256:72279a7f823969e18b10e438950f58330d17e0fcffb57cbd7929770cd687ecb2 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
pytest==4.6.4 \
--hash=sha256:6aa9bc2f6f6504d7949e9df2a756739ca06e58ffda19b5e53c725f7b03fb4aae \
--hash=sha256:b77ae6f2d1a760760902a7676887b665c086f71e3461c64ed2a312afcedc00d6 \
- # via -r requirements.in, pytest-random-order
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in, pytest-random-order
python-dateutil==2.8.2 \
--hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
--hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 \
# via botocore, kubernetes, typepy
python-jenkins==1.4.0 \
--hash=sha256:2782e9c9c66d91bcd43014e8cc649d89218d6a669597878cc350b726fdb8a959 \
--hash=sha256:4fd92d2c90c3534a163945ceb0d5850c26c1a54533b8fed6bd15b2c8998324c0 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
pytz==2021.1 \
--hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \
--hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 \
@@ -714,7 +740,7 @@ pyyaml==5.3.1 \
--hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
--hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
--hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
- # via -r requirements.in, awscli, kubernetes, pre-commit
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in, awscli, kubernetes, pre-commit
pyzmq==19.0.2 \
--hash=sha256:00dca814469436455399660247d74045172955459c0bd49b54a540ce4d652185 \
--hash=sha256:046b92e860914e39612e84fa760fc3f16054d268c11e0e25dcb011fb1bc6a075 \
@@ -748,18 +774,18 @@ pyzmq==19.0.2 \
--hash=sha256:e36f12f503511d72d9bdfae11cadbadca22ff632ff67c1b5459f69756a029c19 \
--hash=sha256:f1a25a61495b6f7bb986accc5b597a3541d9bd3ef0016f50be16dbb32025b302 \
--hash=sha256:fa411b1d8f371d3a49d31b0789eb6da2537dadbb2aef74a43aa99a78195c3f76 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
https://github.com/fruch/repodataParser/archive/py3.zip ; python_version > "3" \
--hash=sha256:3424db354bb58a6bd254546f3499510cb0b5c1543835cd92bc7dfe01b7bf828d \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
requests-oauthlib==1.3.0 \
--hash=sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d \
--hash=sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a \
# via kubernetes, msrest
requests==2.25.1 \
--hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \
--hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e \
- # via -r requirements.in, apache-libcloud, azure-core, docker, google-api-core, kubernetes, msal, msrest, python-jenkins, requests-oauthlib
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in, apache-libcloud, azure-core, docker, google-api-core, kubernetes, msal, msrest, python-jenkins, requests-oauthlib
rsa==4.5 \
--hash=sha256:35c5b5f6675ac02120036d97cf96f1fde4d49670543db2822ba5015e21a18032 \
--hash=sha256:4d409f5a7d78530a4a2062574c7bd80311bc3af29b364e293aa9b03eea77714f \
@@ -771,7 +797,7 @@ s3transfer==0.3.7 \
selenium==3.141.0 \
--hash=sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c \
--hash=sha256:deaf32b60ad91a4611b98d8002757f29e6f2c2d5fcaf202e1c9ad06d6772300d \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
shellingham==1.4.0 \
--hash=sha256:4855c2458d6904829bd34c299f11fdeed7cfefbf8a2c522e4caea6cd76b3171e \
--hash=sha256:536b67a0697f2e4af32ab176c00a50ac2899c5a05e0d8e2dadac8e58888283f9 \
@@ -787,53 +813,57 @@ six==1.16.0 \
sortedcontainers==1.5.9 \
--hash=sha256:844daced0f20d75c02ce53f373d048ea2e401ad8a7b3a4c43b2aa544b569efb3 \
--hash=sha256:fb9e22cd6ee4b459f0d7b9b4189b19631031c72ac05715563139162014c13672 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
sqliteschema==1.2.0 \
--hash=sha256:4c18f141239c58b6dbdc69130be41ed9740d8209a7ab6d529ba982a8b79c071a \
--hash=sha256:e6330fa077ba11f05520acd4f5d5d35596d0b39d71975cf28d529c36e00d2696 \
# via simplesqlite
https://github.com/dkropachev/ssh2-python/archive/fix_segmentation.zip \
--hash=sha256:da673418258b21c8267c042bfdb103fed32b1c664947242f4c79786c03e0c746 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
subprocrunner==1.6.0 \
--hash=sha256:1c1e928db282e7453853744290908d04047a8fb3b49ff1e7f8284e6a3be3a810 \
--hash=sha256:8ed357578c2b11b54ed2bce15fc169adf71349830a11a333d16f6057e33e1c81 \
# via tcconfig
-tabledata==1.2.0 \
- --hash=sha256:0497e408ed3af83e9273d0afa116bde92de89662e67ab4d477118e817eca651a \
- --hash=sha256:6c9c62af9db5246787f722aac6018957c39467e9957b030fa0f89f8a5ca27e9c \
+tabledata==1.3.0 \
+ --hash=sha256:2016fa561552bbf2266682fe328e9161359e605620084bac4754e91c238880f1 \
+ --hash=sha256:54541b0c9e58f8fa38251ea0a60965dbaf95737027fa80e6ab56f98d7e4d61e9 \
# via simplesqlite, sqliteschema
tcconfig==0.26.0 \
--hash=sha256:8c73d868f555e0c316033b6d76d2362828e3f017896918f059e7c8993ff2fe17 \
--hash=sha256:b4b064f96c28f7a171552e2854c04849585fd47aba6df46539a0de776a4eebbc \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
tenacity==5.0.4 \
--hash=sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b \
--hash=sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705 \
- # via -r requirements.in
+ # via -r /extra/scylladb/scylla-cluster-tests/dkropachev6/requirements.in
toml==0.10.2 \
--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
--hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f \
# via autopep8, pre-commit, pylint
-typepy[datetime]==1.2.0 \
- --hash=sha256:96b4c50151ffaca025b7202cdd4e84987ca058f4d6cf1aad0d9c82226961455e \
- --hash=sha256:d3cdabcabf1b6058ccaac1c7ef6a097ec3901a3a3ff79b0a038d8496b3900805 \
+typepy[datetime]==1.3.0 \
+ --hash=sha256:96788530614083164993d1443959f6c58e6bb8e2da839812ddf462c203e4b84c \
+ --hash=sha256:cf1913982969cf6348152c4a5feec08e324addd99670999e57cdb3ad87a61e9a \
# via dataproperty, humanreadable, simplesqlite, sqliteschema, tabledata, tcconfig
+typing-extensions==3.10.0.2 \
+ --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \
+ --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \
+ --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 \
+ # via pydantic
uritemplate==3.0.1 \
--hash=sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f \
--hash=sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae \
# via google-api-python-client
-urllib3==1.26.6 \
- --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \
- --hash=sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f \
+urllib3==1.26.7 \
+ --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \
+ --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844 \
# via botocore, elasticsearch, kubernetes, requests, selenium
-virtualenv==20.7.2 \
- --hash=sha256:9ef4e8ee4710826e98ff3075c9a4739e2cb1040de6a2a8d35db0055840dc96a0 \
- --hash=sha256:e4670891b3a03eb071748c569a87cceaefbf643c5bac46d996c5a45c34aa0f06 \
+virtualenv==20.8.1 \
+ --hash=sha256:10062e34c204b5e4ec5f62e6ef2473f8ba76513a9a617e873f1f8fb4a519d300 \
+ --hash=sha256:bcc17f0b3a29670dd777d6f0755a4c04f28815395bca279cdcb213b97199a6b8 \
# via pre-commit
-voluptuous==0.12.1 \
- --hash=sha256:663572419281ddfaf4b4197fd4942d181630120fb39b333e3adad70aeb56444b \
- --hash=sha256:8ace33fcf9e6b1f59406bfaf6b8ec7bcc44266a9f29080b4deb4fe6ff2492386 \
+voluptuous==0.12.2 \
+ --hash=sha256:4db1ac5079db9249820d49c891cb4660a6f8cae350491210abce741fabf56513 \
# via tcconfig
wcwidth==0.2.5 \
--hash=sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784 \
@@ -856,7 +886,7 @@ pip==21.2.4 \
--hash=sha256:0eb8a1516c3d138ae8689c0c1a60fde7143310832f9dc77e11d8a4bc62de193b \
--hash=sha256:fa9ebb85d3fd607617c0c44aca302b1b45d87f9c2a1649b46c26167ca4296323 \
# via pip-tools
-setuptools==58.0.4 \
- --hash=sha256:69cc739bc2662098a68a9bc575cd974a57969e70c1d58ade89d104ab73d79770 \
- --hash=sha256:f10059f0152e0b7fb6b2edd77bcb1ecd4c9ed7048a826eb2d79f72fd2e6e237b \
+setuptools==58.1.0 \
+ --hash=sha256:5de67252090e08d25f240f07d80310f778a5a46cdcf9ea9855662630ac8547b2 \
+ --hash=sha256:7324fd4b66efa05cdfc9c89174573a4410acc7848f318cc0565c7fb659dfdc81 \
# via anyconfig, google-api-core, google-auth, kubernetes, tcconfig
diff --git a/sdcm/provision/__init__.py b/sdcm/provision/__init__.py
--- a/sdcm/provision/__init__.py
+++ b/sdcm/provision/__init__.py
@@ -0,0 +1,12 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
diff --git a/sdcm/provision/common/__init__.py b/sdcm/provision/common/__init__.py
--- a/sdcm/provision/common/__init__.py
+++ b/sdcm/provision/common/__init__.py
@@ -0,0 +1,12 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
diff --git a/sdcm/provision/common/builders.py b/sdcm/provision/common/builders.py
--- a/sdcm/provision/common/builders.py
+++ b/sdcm/provision/common/builders.py
@@ -0,0 +1,74 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
+from typing import Union
+
+from sdcm.provision.common.utils import BaseModel
+
+
+OptionalType = type(Union[str, None])
+
+
+class AttrBuilder(BaseModel):
+ @classmethod
+ def get_properties(cls):
+ return [prop for prop in dir(cls) if isinstance(getattr(cls, prop), property) and prop[0] != '_']
+
+ @property
+ def _exclude_by_default(self):
+ exclude_fields = []
+ for field_name, field in self.__fields__.items():
+ if not field.field_info.extra.get('as_dict', True):
+ exclude_fields.append(field_name)
+ return set(exclude_fields)
+
+ def dict(
+ self,
+ *,
+ include: Union['AbstractSetIntStr', 'MappingIntStrAny'] = None,
+ exclude: Union['AbstractSetIntStr', 'MappingIntStrAny'] = None,
+ by_alias: bool = False,
+ skip_defaults: bool = None,
+ exclude_unset: bool = False,
+ exclude_defaults: bool = False,
+ exclude_none: bool = False,
+ ) -> 'DictStrAny':
+ """
+ Pydantic does not treat properties as fields, so you can't get their values when call dict
+ This function is to enable property extraction
+ """
+ if exclude is None:
+ exclude = self._exclude_by_default
+ attribs = super().dict(
+ include=include,
+ exclude=exclude,
+ by_alias=by_alias,
+ skip_defaults=skip_defaults,
+ exclude_unset=exclude_unset,
+ exclude_defaults=exclude_defaults,
+ exclude_none=exclude_none
+ )
+ props = self.get_properties()
+ if include:
+ props = [prop for prop in props if prop in include]
+ if exclude:
+ props = [prop for prop in props if prop not in exclude]
+ if props:
+ if exclude_unset or exclude_none or exclude_defaults:
+ for prop in props:
+ prop_value = getattr(self, prop)
+ if prop_value is not None:
+ attribs[prop] = prop_value
+ else:
+ attribs.update({prop: getattr(self, prop) for prop in props})
+ return attribs
diff --git a/sdcm/provision/common/utils.py b/sdcm/provision/common/utils.py
--- a/sdcm/provision/common/utils.py
+++ b/sdcm/provision/common/utils.py
@@ -0,0 +1,77 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+from typing import Any, List, Dict
+
+import attr
+
+from pydantic import BaseModel as PyBaseModel # pylint: disable=no-name-in-module
+
+
+__builtin__ = [].__class__.__module__
+AttrAttributeType = type(attr.attrib())
+
+
+def is_builtin(inst):
+ return inst.__module__ == __builtin__
+
+
+def instance_to_dict_without_defaults(inst):
+ return attr.asdict(inst, filter=lambda attr_info, value: attr_info.default != value)
+
+
+def instance_to_dict_with_defaults(inst):
+ return attr.asdict(inst)
+
+
+def as_dict(data) -> Any: # pylint: disable=too-many-branches
+ if hasattr(data, "__attrs_attrs__"):
+ fields = attr.fields_dict(type(data))
+ output: Dict[str, Any] = {}
+ for attr_name in dir(data):
+ if attr_name[0] == '_':
+ continue
+ attr_value = getattr(data, attr_name)
+ if attr_annotation := fields.get(attr_name):
+ if not attr_annotation.metadata.get('as_dict', True) or attr_value == attr_annotation.default:
+ continue
+ if attr_value is None or callable(attr_value):
+ continue
+ if isinstance(attr_value, (str, float, int)):
+ output[attr_name] = attr_value
+ else:
+ output[attr_name] = as_dict(attr_value)
+ return output
+ if isinstance(data, (list, tuple)):
+ output: List[Any] = [None] * len(data)
+ for attr_name, attr_value in enumerate(data):
+ if isinstance(attr_value, (str, float, int)):
+ output[attr_name] = attr_value
+ else:
+ output[attr_name] = as_dict(attr_value)
+ return output
+ if isinstance(data, dict):
+ output = {}
+ for attr_name, attr_value in data.items():
+ if isinstance(attr_value, (str, float, int)):
+ output[attr_name] = attr_value
+ elif hasattr(attr_value, 'as_dict'):
+ output[attr_name] = attr_value.as_dict()
+ else:
+ output[attr_name] = as_dict(attr_value)
+ return output
+ raise ValueError("Unexpected data type %s" % (type(data),))
+
+
+class BaseModel(PyBaseModel): # pylint: disable=too-few-public-methods
+ def __init__(self, **kwargs):
+ super().__init__(**{name: value for name, value in kwargs.items() if value is not None})
diff --git a/sdcm/provision/helpers/certificate.py b/sdcm/provision/helpers/certificate.py
--- a/sdcm/provision/helpers/certificate.py
+++ b/sdcm/provision/helpers/certificate.py
@@ -0,0 +1,30 @@
+from textwrap import dedent
+
+from sdcm.remote import shell_script_cmd
+
+
+def install_client_certificate(remoter):
+ if remoter.run('ls /etc/scylla/ssl_conf', ignore_status=True).ok:
+ return
+ remoter.send_files(src='./data_dir/ssl_conf', dst='/tmp/') # pylint: disable=not-callable
+ setup_script = dedent("""
+ mkdir -p ~/.cassandra/
+ cp /tmp/ssl_conf/client/cqlshrc ~/.cassandra/
+ sudo mkdir -p /etc/scylla/
+ sudo rm -rf /etc/scylla/ssl_conf/
+ sudo mv -f /tmp/ssl_conf/ /etc/scylla/
+ """)
+ remoter.run('bash -cxe "%s"' % setup_script)
+
+
+def install_encryption_at_rest_files(remoter):
+ if remoter.sudo('ls /etc/encrypt_conf/system_key_dir', ignore_status=True).ok:
+ return
+ remoter.send_files(src="./data_dir/encrypt_conf", dst="/tmp/")
+ remoter.sudo(shell_script_cmd(dedent("""
+ rm -rf /etc/encrypt_conf
+ mv -f /tmp/encrypt_conf /etc
+ mkdir -p /etc/scylla/encrypt_conf /etc/encrypt_conf/system_key_dir
+ chown -R scylla:scylla /etc/scylla /etc/encrypt_conf
+ """)))
+ remoter.sudo("md5sum /etc/encrypt_conf/*.pem", ignore_status=True)
diff --git a/sdcm/provision/scylla_yaml/__init__.py b/sdcm/provision/scylla_yaml/__init__.py
--- a/sdcm/provision/scylla_yaml/__init__.py
+++ b/sdcm/provision/scylla_yaml/__init__.py
@@ -0,0 +1,18 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
+from .scylla_yaml import ScyllaYaml
+from .auxiliaries import ServerEncryptionOptions, ClientEncryptionOptions, SeedProvider, RequestSchedulerOptions
+from .certificate_builder import ScyllaYamlCertificateAttrBuilder
+from .cluster_builder import ScyllaYamlClusterAttrBuilder
+from .node_builder import ScyllaYamlNodeAttrBuilder
diff --git a/sdcm/provision/scylla_yaml/auxiliaries.py b/sdcm/provision/scylla_yaml/auxiliaries.py
--- a/sdcm/provision/scylla_yaml/auxiliaries.py
+++ b/sdcm/provision/scylla_yaml/auxiliaries.py
@@ -0,0 +1,158 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
+
+import logging
+from typing import Literal, List, Union, Optional
+
+from pydantic import Field, validator
+
+from sdcm.provision.common.builders import AttrBuilder
+from sdcm.provision.common.utils import BaseModel
+from sdcm.sct_config import SCTConfiguration
+
+
+SEED_PROVIDERS = [
+ 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'org.apache.cassandra.locator.GossipingPropertyFileSnitch',
+ 'org.apache.cassandra.locator.Ec2Snitch',
+ 'org.apache.cassandra.locator.Ec2MultiRegionSnitch',
+ 'org.apache.cassandra.locator.GoogleCloudSnitch',
+ 'org.apache.cassandra.locator.RackInferringSnitch',
+]
+SASLAUTHD_AUTHENTICATOR = 'com.scylladb.auth.SaslauthdAuthenticator'
+LOGGER = logging.getLogger(__file__)
+
+
+class SeedProvider(BaseModel): # pylint: disable=too-few-public-methods
+ class_name: Literal[
+ 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'org.apache.cassandra.locator.GossipingPropertyFileSnitch',
+ 'org.apache.cassandra.locator.Ec2Snitch',
+ 'org.apache.cassandra.locator.Ec2MultiRegionSnitch',
+ 'org.apache.cassandra.locator.GoogleCloudSnitch',
+ 'org.apache.cassandra.locator.RackInferringSnitch',
+ ]
+ parameters: List[dict] = None
+
+ # pylint: disable=no-self-argument,no-self-use
+ @validator("class_name", pre=True, always=True)
+ def set_class_name(cls, class_name: str):
+ if class_name.startswith('org.apache.cassandra.locator.'):
+ return class_name
+ return 'org.apache.cassandra.locator.' + class_name
+
+
+class ServerEncryptionOptions(BaseModel): # pylint: disable=too-few-public-methods
+ internode_encryption: Literal['all', 'none', 'dc', 'rack'] = 'none'
+ certificate: str = 'conf/scylla.crt'
+ keyfile: str = 'conf/scylla.key'
+ truststore: str = None
+ priority_string: str = None
+ require_client_auth: bool = False
+
+
+class ClientEncryptionOptions(BaseModel): # pylint: disable=too-few-public-methods
+ enabled: bool = False
+ certificate: str = 'conf/scylla.crt'
+ keyfile: str = 'conf/scylla.key'
+ truststore: str = None
+ priority_string: str = None
+ require_client_auth: bool = False
+
+
+class RequestSchedulerOptions(BaseModel): # pylint: disable=too-few-public-methods
+ throttle_limit: int = None
+ default_weight: int = 1
+ weights: int = 1
+
+
+EndPointSnitchType = Literal[
+ 'org.apache.cassandra.locator.SimpleSnitch',
+ 'org.apache.cassandra.locator.GossipingPropertyFileSnitch',
+ 'org.apache.cassandra.locator.PropertyFileSnitch',
+ 'org.apache.cassandra.locator.Ec2Snitch',
+ 'org.apache.cassandra.locator.Ec2MultiRegionSnitch',
+ 'org.apache.cassandra.locator.RackInferringSnitch'
+]
+
+
+class ScyllaYamlAttrBuilderBase(AttrBuilder):
+ params: Union[SCTConfiguration, dict] = Field(as_dict=False)
+
+ @property
+ def _cluster_backend(self) -> str:
+ return self.params.get('cluster_backend')
+
+ @property
+ def _cloud_provider(self) -> Literal['aws', 'gce', 'azure', None]:
+ for provider in ['aws', 'gce', 'azure']:
+ if provider in self._cluster_backend:
+ return provider
+ if self._cluster_backend == "k8s-eks":
+ return 'aws'
+ if self._cluster_backend == "k8s-gke":
+ return 'gce'
+ return None
+
+ @property
+ def _region(self) -> str:
+ if self._cloud_provider == 'aws':
+ return self.params.get('region_name')
+ elif self._cloud_provider == 'gce':
+ return self.params.get('gce_datacenter')
+ elif self._cloud_provider == 'azure':
+ return self.params.get('region_name')
+ return ''
+
+ @property
+ def _multi_region(self) -> bool:
+ """
+ Analog of TestConfig.MULTI_REGION
+ """
+ if not self._region:
+ return False
+ return len(self._region.split()) > 1 # pylint: disable=no-member
+
+ @property
+ def _intra_node_comm_public(self) -> bool:
+ """
+ Analog of TestConfig.INTRA_NODE_COMM_PUBLIC
+ """
+ return self.params.get('intra_node_comm_public') or self._multi_region
+
+ @property
+ def _authenticator(self) -> Optional[str]:
+ return self.params.get('authenticator')
+
+ @property
+ def _is_authenticator_valid(self) -> bool:
+ return self._authenticator in ['AllowAllAuthenticator', 'PasswordAuthenticator', SASLAUTHD_AUTHENTICATOR]
+
+ @property
+ def _authorizer(self) -> Optional[str]:
+ return self.params.get('authorizer')
+
+ @property
+ def _is_ip_ssh_connections_ipv6(self):
+ if self.params.get('ip_ssh_connections') == 'ipv6':
+ return True
+ return False
+
+ @property
+ def _default_endpoint_snitch(self) -> Literal[
+ 'org.apache.cassandra.locator.Ec2MultiRegionSnitch',
+ 'org.apache.cassandra.locator.GossipingPropertyFileSnitch']:
+ if self._cluster_backend == 'aws':
+ return 'org.apache.cassandra.locator.Ec2MultiRegionSnitch'
+ return 'org.apache.cassandra.locator.GossipingPropertyFileSnitch'
diff --git a/sdcm/provision/scylla_yaml/certificate_builder.py b/sdcm/provision/scylla_yaml/certificate_builder.py
--- a/sdcm/provision/scylla_yaml/certificate_builder.py
+++ b/sdcm/provision/scylla_yaml/certificate_builder.py
@@ -0,0 +1,53 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
+from functools import cached_property
+from typing import Optional, Any
+
+from sdcm.provision.helpers.certificate import install_client_certificate
+from sdcm.provision.scylla_yaml.auxiliaries import ScyllaYamlAttrBuilderBase, ClientEncryptionOptions, \
+ ServerEncryptionOptions
+
+
+class ScyllaYamlCertificateAttrBuilder(ScyllaYamlAttrBuilderBase):
+ """
+ Builds scylla yaml attributes regarding encryption
+ """
+ node: Any
+
+ @cached_property
+ def _ssl_files_path(self) -> str:
+ install_client_certificate(self.node.remoter)
+ return '/etc/scylla/ssl_conf'
+
+ @property
+ def client_encryption_options(self) -> Optional[ClientEncryptionOptions]:
+ if not self.params.get('client_encrypt'):
+ return None
+ return ClientEncryptionOptions(
+ enabled=True,
+ certificate=self._ssl_files_path + '/client/test.crt',
+ keyfile=self._ssl_files_path + '/client/test.key',
+ truststore=self._ssl_files_path + '/client/catest.pem',
+ )
+
+ @property
+ def server_encryption_options(self) -> Optional[ServerEncryptionOptions]:
+ if not self.params.get('internode_encryption') or not self.params.get('server_encrypt'):
+ return None
+ return ServerEncryptionOptions(
+ internode_encryption=self.params.get('internode_encryption'),
+ certificate=self._ssl_files_path + '/db.crt',
+ keyfile=self._ssl_files_path + '/db.key',
+ truststore=self._ssl_files_path + '/cadb.pem',
+ )
diff --git a/sdcm/provision/scylla_yaml/cluster_builder.py b/sdcm/provision/scylla_yaml/cluster_builder.py
--- a/sdcm/provision/scylla_yaml/cluster_builder.py
+++ b/sdcm/provision/scylla_yaml/cluster_builder.py
@@ -0,0 +1,189 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
+from functools import cached_property
+from typing import Optional, Any
+
+from pydantic import Field
+
+from sdcm.provision.scylla_yaml.auxiliaries import ScyllaYamlAttrBuilderBase
+from sdcm.utils.ldap import LDAP_SSH_TUNNEL_LOCAL_PORT
+
+
+class ScyllaYamlClusterAttrBuilder(ScyllaYamlAttrBuilderBase):
+ """
+ Builds scylla yaml attributes that stays persistent across the cluster
+ """
+
+ test_config: Any = Field(as_dict=False)
+ msldap_server_info: dict = Field(as_dict=False, default=None)
+
+ @property
+ def hinted_handoff_enabled(self) -> Optional[str]:
+ if self.params.get('hinted_handoff') == 'enabled':
+ return 'enabled'
+ if self.params.get('hinted_handoff') == 'disabled':
+ return 'disabled'
+ return None
+
+ @property
+ def experimental(self) -> Optional[bool]:
+ if self.params.get('experimental') is None:
+ return None
+ return bool(self.params.get('experimental'))
+
+ @property
+ def enable_ipv6_dns_lookup(self) -> bool:
+ return self._is_ip_ssh_connections_ipv6
+
+ @property
+ def authenticator(self) -> Optional[str]:
+ if self._is_authenticator_valid:
+ return self._authenticator
+ return None
+
+ @property
+ def saslauthd_socket_path(self) -> Optional[str]:
+ if self._is_authenticator_valid:
+ return '/run/saslauthd/mux'
+ return None
+
+ @property
+ def authorizer(self) -> Optional[str]:
+ if self._authorizer in ['AllowAllAuthorizer', 'CassandraAuthorizer']:
+ return self._authorizer
+ return None
+
+ @property
+ def alternator_port(self) -> Optional[str]:
+ return self.params.get('alternator_port')
+
+ @property
+ def alternator_write_isolation(self) -> Optional[str]:
+ return "always_use_lwt" if self.params.get('alternator_port') else None
+
+ @property
+ def alternator_enforce_authorization(self) -> bool:
+ return bool(self.params.get('alternator_enforce_authorization'))
+
+ @property
+ def internode_compression(self) -> Optional[str]:
+ return self.params.get('internode_compression')
+
+ @property
+ def endpoint_snitch(self) -> Optional[str]:
+ """
+ Comes from get_endpoint_snitch
+ """
+ if snitch := self.params.get('endpoint_snitch'):
+ return snitch
+ if self._multi_region:
+ return self._default_endpoint_snitch
+ return None
+
+ @property
+ def ldap_attr_role(self) -> Optional[str]:
+ return 'cn' if self._is_ldap_authorization else None
+
+ @property
+ def ldap_bind_dn(self) -> Optional[str]:
+ if self._is_msldap_authorization:
+ return self._ms_ldap_bind_dn
+ if self._is_openldap_authorization:
+ return self._open_ldap_bind_dn
+ return None
+
+ @property
+ def ldap_bind_passwd(self) -> Optional[str]:
+ if self._is_msldap_authorization:
+ return self._ms_ldap_bind_passwd
+ if self._is_openldap_authorization:
+ return self._open_ldap_bind_passwd
+ return None
+
+ @property
+ def ldap_url_template(self) -> Optional[str]:
+ if self._is_msldap_authorization:
+ server_port = self._ms_ldap_server_address_port
+ ldap_filter = 'member=CN={USER}'
+ elif self._is_openldap_authorization:
+ server_port = self._open_ldap_server_address_port
+ ldap_filter = 'uniqueMember=uid={USER},ou=Person'
+ else:
+ return None
+ return f'ldap://{server_port}/{self._ldap_base_dn}?cn?sub?({ldap_filter},{self._ldap_base_dn})'
+
+ @property
+ def _is_msldap_authorization(self):
+ return self._is_ldap_authorization and self.params.get('use_ms_ad_ldap')
+
+ @property
+ def _is_openldap_authorization(self):
+ return self._is_ldap_authorization and not self.params.get('use_ms_ad_ldap')
+
+ @property
+ def _is_ldap_authorization(self):
+ return self.params.get('use_ldap_authorization')
+
+ @property
+ def _ldap_base_dn(self) -> str:
+ return 'dc=scylla-qa,dc=com'
+
+ @property
+ def _ms_ldap_base_dn(self) -> str:
+ return 'DC=scylla-qa,DC=com'
+
+ @property
+ def _ms_ldap_bind_dn(self) -> str:
+ return self._msldap_server_info['ldap_bind_dn']
+
+ @property
+ def _ms_ldap_bind_passwd(self) -> str:
+ return self._msldap_server_info['admin_password']
+
+ @property
+ def _ms_ldap_server_address_port(self) -> str:
+ return f'{self._msldap_server_info["server_address"]}:389'
+
+ @property
+ def _open_ldap_bind_dn(self) -> str:
+ return f'cn=admin,{self._ldap_base_dn}'
+
+ @property
+ def _open_ldap_bind_passwd(self) -> str:
+ return 'scylla-0'
+
+ @property
+ def _open_ldap_server_address_port(self) -> str:
+ if self.params.get('ip_ssh_connections') == 'public' or self._multi_region:
+ # When connection goes public we run ssh tunnel on db-nodes side to access openldap server
+ # that is why we pass address it in scylla config as '127.0.0.1:{LDAP_SSH_TUNNEL_LOCAL_PORT}'
+ return '127.0.0.1:' + str(LDAP_SSH_TUNNEL_LOCAL_PORT)
+ return self._openldap_server_address_port
+
+ @cached_property
+ def _openldap_server_address_port(self) -> str:
+ if (not self.test_config.LDAP_ADDRESS or
+ not self.test_config.LDAP_ADDRESS[0] or
+ not self.test_config.LDAP_ADDRESS[1]):
+ raise RuntimeError("OPENLDAP has not been started")
+ return str(self.test_config.LDAP_ADDRESS[0]) + ':' + str(self.test_config.LDAP_ADDRESS[1])
+
+ @cached_property
+ def _msldap_server_info(self) -> dict:
+ if not self.msldap_server_info:
+ raise RuntimeError('MSLDAP is configured, but not `msldap_server_info` is provided')
+ if missing_keys := {'ldap_bind_dn', 'admin_password', 'server_address'} - set(self.msldap_server_info.keys()):
+ raise RuntimeError("MSLDAP is configured, but `msldap_server_info` lack of following keys: "
+ f"{','.join(missing_keys)}")
+ return self.msldap_server_info
diff --git a/sdcm/provision/scylla_yaml/node_builder.py b/sdcm/provision/scylla_yaml/node_builder.py
--- a/sdcm/provision/scylla_yaml/node_builder.py
+++ b/sdcm/provision/scylla_yaml/node_builder.py
@@ -0,0 +1,99 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
+
+from typing import Optional, List, Any
+
+from pydantic import Field
+
+from sdcm.provision.scylla_yaml.auxiliaries import ScyllaYamlAttrBuilderBase, SeedProvider
+
+
+class ScyllaYamlNodeAttrBuilder(ScyllaYamlAttrBuilderBase):
+ """
+ Builds scylla yaml attributes that are needed to keep node connected to the other nodes in the cluster
+ """
+ node: Any = Field(as_dict=False)
+
+ @property
+ def _seed_address(self) -> str:
+ return ','.join(self.node.parent_cluster.seed_nodes_ips)
+
+ @property
+ def _private_ip_address(self) -> str:
+ return self.node.private_ip_address
+
+ @property
+ def _public_ip_address(self) -> str:
+ return self.node.public_ip_address
+
+ @property
+ def _ipv6_ip_address(self) -> str:
+ return self.node.ipv6_ip_address
+
+ @property
+ def seed_provider(self) -> Optional[List[SeedProvider]]:
+ if not self._seed_address:
+ return None
+ return [
+ SeedProvider(
+ class_name='org.apache.cassandra.locator.SimpleSeedProvider',
+ parameters=[{'seeds': self._seed_address}]
+ )
+ ]
+
+ @property
+ def listen_address(self) -> Optional[str]:
+ if self._is_ip_ssh_connections_ipv6:
+ return self._ipv6_ip_address
+ if self.params.get('extra_network_interface'):
+ # Scylla should be listening on all interfaces
+ return '0.0.0.0'
+ return self._private_ip_address
+
+ @property
+ def rpc_address(self) -> Optional[str]:
+ if self._is_ip_ssh_connections_ipv6:
+ return self._ipv6_ip_address
+ if self.params.get('extra_network_interface'):
+ # Scylla should be listening on all interfaces
+ return '0.0.0.0'
+ return self._private_ip_address
+
+ @property
+ def broadcast_rpc_address(self) -> Optional[str]:
+ if self._is_ip_ssh_connections_ipv6:
+ return self._ipv6_ip_address
+ if self.params.get('extra_network_interface'):
+ # Scylla should be listening on all interfaces
+ return self._private_ip_address
+ if self._intra_node_comm_public:
+ return self._public_ip_address
+ return None
+
+ @property
+ def broadcast_address(self) -> Optional[str]:
+ if self._is_ip_ssh_connections_ipv6:
+ return self._ipv6_ip_address
+ if self.params.get('extra_network_interface'):
+ # Scylla should be listening on all interfaces
+ return self._private_ip_address
+ if self._intra_node_comm_public:
+ return self._public_ip_address
+ return None
+
+ @property
+ def prometheus_address(self) -> Optional[str]:
+ if self._is_ip_ssh_connections_ipv6:
+ return self._ipv6_ip_address
+ return None
diff --git a/sdcm/provision/scylla_yaml/scylla_yaml.py b/sdcm/provision/scylla_yaml/scylla_yaml.py
--- a/sdcm/provision/scylla_yaml/scylla_yaml.py
+++ b/sdcm/provision/scylla_yaml/scylla_yaml.py
@@ -0,0 +1,343 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
+
+from typing import List, Literal, Union
+
+from pydantic import Field, validator
+
+from sdcm.provision.common.utils import BaseModel
+from sdcm.provision.scylla_yaml.auxiliaries import RequestSchedulerOptions, EndPointSnitchType, SeedProvider, \
+ ServerEncryptionOptions, ClientEncryptionOptions
+
+
+class ScyllaYaml(BaseModel): # pylint: disable=too-few-public-methods
+ broadcast_address: str = ""
+ api_port: int = 10000
+ api_address: str = ""
+ ssl_storage_port: int = 7001
+ background_writer_scheduling_quota: float = 1.0
+ auto_adjust_flush_quota: bool = False
+ memtable_flush_static_shares: int = 0
+ compaction_static_shares: int = 0
+ compaction_enforce_min_threshold: bool = False
+ cluster_name: str = ""
+ listen_address: str = "localhost"
+ listen_interface: str = "eth0"
+ listen_interface_prefer_ipv6: bool = False
+ commitlog_directory: str = ""
+ data_file_directories: List[str] = None
+ hints_directory: str = ""
+ view_hints_directory: str = ""
+ saved_caches_directory: str = ""
+ commit_failure_policy: Literal["die", "stop", "stop_commit", "ignore"] = "stop"
+ disk_failure_policy: Literal["die", "stop_paranoid", "stop", "best_effort", "ignore"] = "stop"
+ endpoint_snitch: EndPointSnitchType = Field('org.apache.cassandra.locator.SimpleSnitch')
+
+ # pylint: disable=no-self-argument,no-self-use
+ @validator("endpoint_snitch", pre=True, always=True)
+ def set_endpoint_snitch(cls, endpoint_snitch: str):
+ if endpoint_snitch.startswith('org.apache.cassandra.locator.'):
+ return endpoint_snitch
+ return 'org.apache.cassandra.locator.' + endpoint_snitch
+
+ rpc_address: str = 'localhost'
+ rpc_interface: str = "eth1"
+ rpc_interface_prefer_ipv6: bool = False
+ seed_provider: List[SeedProvider] = [SeedProvider(class_name='org.apache.cassandra.locator.SimpleSeedProvider')]
+ compaction_throughput_mb_per_sec: int = 16
+ compaction_large_partition_warning_threshold_mb: int = 1000
+ compaction_large_row_warning_threshold_mb: int = 10
+ compaction_large_cell_warning_threshold_mb: int = 1
+ compaction_rows_count_warning_threshold: int = 100000
+ memtable_total_space_in_mb: int = 0
+ concurrent_reads: int = 32
+ concurrent_writes: int = 32
+ concurrent_counter_writes: int = 32
+ incremental_backups: bool = False
+ snapshot_before_compaction: bool = False
+ phi_convict_threshold: int = 8
+ commitlog_sync: Literal['periodic', 'batch'] = 'periodic'
+ commitlog_segment_size_in_mb: int = 64
+ commitlog_sync_period_in_ms: int = 10000
+ commitlog_sync_batch_window_in_ms: int = 10000
+ commitlog_total_space_in_mb: int = -1
+ commitlog_reuse_segments: bool = True
+ commitlog_use_o_dsync: bool = True
+ compaction_preheat_key_cache: bool = True
+ concurrent_compactors: int = 0
+ in_memory_compaction_limit_in_mb: int = 64
+ preheat_kernel_page_cache: bool = False
+ sstable_preemptive_open_interval_in_mb: int = 50
+ defragment_memory_on_idle: bool = False
+ memtable_allocation_type: Literal[
+ 'heap_buffers',
+ 'offheap_buffers',
+ 'offheap_objects'
+ ] = "heap_buffers"
+ memtable_cleanup_threshold: float = .11
+ file_cache_size_in_mb: int = 512
+ memtable_flush_queue_size: int = 4
+ memtable_flush_writers: int = 1
+ memtable_heap_space_in_mb: int = 0
+ memtable_offheap_space_in_mb: int = 0
+ column_index_size_in_kb: int = 64
+ index_summary_capacity_in_mb: int = 0
+ index_summary_resize_interval_in_minutes: int = 60
+ reduce_cache_capacity_to: float = .6
+ reduce_cache_sizes_at: float = .85
+ stream_throughput_outbound_megabits_per_sec: int = 400
+ inter_dc_stream_throughput_outbound_megabits_per_sec: int = 0
+ trickle_fsync: bool = False
+ trickle_fsync_interval_in_kb: int = 10240
+ auto_bootstrap: bool = True
+ batch_size_warn_threshold_in_kb: int = 5
+ batch_size_fail_threshold_in_kb: int = 50
+ listen_on_broadcast_address: bool = False
+ initial_token: int = None
+ num_tokens: int = 1
+ partitioner: Literal['org.apache.cassandra.dht.Murmur3Partitioner'] = 'org.apache.cassandra.dht.Murmur3Partitioner'
+ storage_port: int = 7000
+ auto_snapshot: bool = True
+ key_cache_keys_to_save: int = 0
+ key_cache_save_period: int = 14400
+ key_cache_size_in_mb: int = 100
+ row_cache_keys_to_save: int = 0
+ row_cache_size_in_mb: int = 0
+ row_cache_save_period: int = 0
+ memory_allocator: Literal[
+ "NativeAllocator",
+ "JEMallocAllocator"
+ ] = "NativeAllocator"
+ counter_cache_size_in_mb: int = 0
+ counter_cache_save_period: int = 7200
+ counter_cache_keys_to_save: int = 0
+ tombstone_warn_threshold: int = 1000
+ tombstone_failure_threshold: int = 100000
+ range_request_timeout_in_ms: int = 10000
+ read_request_timeout_in_ms: int = 5000
+ counter_write_request_timeout_in_ms: int = 5000
+ cas_contention_timeout_in_ms: int = 1000
+ truncate_request_timeout_in_ms: int = 60000
+ write_request_timeout_in_ms: int = 2000
+ request_timeout_in_ms: int = 10000
+ cross_node_timeout: bool = False
+ internode_send_buff_size_in_bytes: int = 0
+ internode_recv_buff_size_in_bytes: int = 0
+ internode_compression: Literal['none', 'all', 'dc'] = "none"
+ inter_dc_tcp_nodelay: bool = False
+ streaming_socket_timeout_in_ms: int = 0
+ start_native_transport: bool = True
+ native_transport_port: int = 9042
+ native_transport_port_ssl: int = 9142
+ native_transport_max_threads: int = 128
+ native_transport_max_frame_size_in_mb: int = 256
+ native_shard_aware_transport_port: int = 19042
+ native_shard_aware_transport_port_ssl: int = 19142
+ broadcast_rpc_address: str = None
+ rpc_port: int = 9160
+ start_rpc: bool = True
+ rpc_keepalive: bool = True
+ rpc_max_threads: int = 0
+ rpc_min_threads: int = 16
+ rpc_recv_buff_size_in_bytes: int = 0
+ rpc_send_buff_size_in_bytes: int = 0
+ rpc_server_type: Literal['sync', 'hsh'] = "sync"
+ cache_hit_rate_read_balancing: bool = True
+ dynamic_snitch_badness_threshold: int = 0
+ dynamic_snitch_reset_interval_in_ms: int = 60000
+ dynamic_snitch_update_interval_in_ms: int = 100
+ hinted_handoff_enabled: Literal['enabled', 'disabled'] = 'enabled'
+ hinted_handoff_throttle_in_kb: int = 1024
+ max_hint_window_in_ms: int = 10800000
+ max_hints_delivery_threads: int = 2
+ batchlog_replay_throttle_in_kb: int = 1024
+ request_scheduler: Literal[
+ 'org.apache.cassandra.scheduler.NoScheduler',
+ 'org.apache.cassandra.scheduler.RoundRobinScheduler'
+ ] = "org.apache.cassandra.scheduler.NoScheduler"
+
+ # pylint: disable=no-self-argument,no-self-use
+ @validator("request_scheduler", pre=True, always=True)
+ def set_request_scheduler(cls, request_scheduler: str):
+ if request_scheduler.startswith('org.apache.cassandra.scheduler.'):
+ return request_scheduler
+ return 'org.apache.cassandra.scheduler.' + request_scheduler
+
+ request_scheduler_id: str = None
+ request_scheduler_options: RequestSchedulerOptions = None
+ thrift_framed_transport_size_in_mb: int = 15
+ thrift_max_message_length_in_mb: int = 16
+ authenticator: Literal[
+ "org.apache.cassandra.auth.PasswordAuthenticator",
+ "org.apache.cassandra.auth.AllowAllAuthenticator",
+ "com.scylladb.auth.TransitionalAuthenticator",
+ "com.scylladb.auth.SaslauthdAuthenticator"
+ ] = "org.apache.cassandra.auth.AllowAllAuthenticator"
+
+ # pylint: disable=no-self-argument,no-self-use
+ @validator("authenticator", pre=True, always=True)
+ def set_authenticator(cls, authenticator: str):
+ if authenticator.startswith('org.apache.cassandra.auth.') or authenticator.startswith('com.scylladb.auth.'):
+ return authenticator
+ if authenticator in ['PasswordAuthenticator', 'AllowAllAuthenticator']:
+ return 'org.apache.cassandra.auth.' + authenticator
+ if authenticator in ['TransitionalAuthenticator', 'SaslauthdAuthenticator']:
+ return 'com.scylladb.auth.' + authenticator
+ return authenticator
+
+ internode_authenticator: Literal["enabled", "disabled"] = "disabled"
+ authorizer: Literal[
+ "org.apache.cassandra.auth.AllowAllAuthorizer",
+ "org.apache.cassandra.auth.CassandraAuthorizer",
+ "com.scylladb.auth.TransitionalAuthorizer",
+ "com.scylladb.auth.SaslauthdAuthorizer"
+ ] = "org.apache.cassandra.auth.AllowAllAuthorizer"
+
+ # pylint: disable=no-self-argument,no-self-use
+ @validator("authorizer", pre=True, always=True)
+ def set_authorizer(cls, authorizer: str):
+ if authorizer.startswith('org.apache.cassandra.auth.') or authorizer.startswith('com.scylladb.auth.'):
+ return authorizer
+ if authorizer in ['AllowAllAuthorizer', 'CassandraAuthorizer']:
+ return 'org.apache.cassandra.auth.' + authorizer
+ if authorizer in ['TransitionalAuthorizer', 'SaslauthdAuthorizer']:
+ return 'com.scylladb.auth.' + authorizer
+ return authorizer
+
+ role_manager: str = "org.apache.cassandra.auth.CassandraRoleManager"
+ permissions_validity_in_ms: int = 10000
+ permissions_update_interval_in_ms: int = 2000
+ permissions_cache_max_entries: int = 1000
+ server_encryption_options: ServerEncryptionOptions = None
+ client_encryption_options: ClientEncryptionOptions = None
+ enable_in_memory_data_store: bool = False
+ enable_cache: bool = True
+ enable_commitlog: bool = True
+ volatile_system_keyspace_for_testing: bool = False
+ api_ui_dir: str = "swagger-ui/dist/"
+ api_doc_dir: str = "api/api-doc/"
+ load_balance: str = "none"
+ consistent_rangemovement: bool = True
+ join_ring: bool = True
+ load_ring_state: bool = True
+ replace_node: str = ""
+ replace_token: str = ""
+ replace_address: str = ""
+ replace_address_first_boot: str = ""
+ override_decommission: bool = False
+ enable_repair_based_node_ops: bool = True
+ ring_delay_ms: int = 30 * 1000
+ shadow_round_ms: int = 300 * 1000
+ fd_max_interval_ms: int = 2 * 1000
+ fd_initial_value_ms: int = 2 * 1000
+ shutdown_announce_in_ms: int = 2 * 1000
+ developer_mode: bool = False
+ skip_wait_for_gossip_to_settle: int = -1
+ force_gossip_generation: int = -1
+ experimental: bool = False
+ experimental_features: dict = {}
+ lsa_reclamation_step: int = 1
+ prometheus_port: int = 9180
+ prometheus_address: str = "0.0.0.0"
+ prometheus_prefix: str = "scylla"
+ abort_on_lsa_bad_alloc: bool = False
+ murmur3_partitioner_ignore_msb_bits: int = 12
+ virtual_dirty_soft_limit: float = 0.6
+ sstable_summary_ratio: float = 0.0005
+ large_memory_allocation_warning_threshold: int = 2 ** 20
+ enable_deprecated_partitioners: bool = False
+ enable_keyspace_column_family_metrics: bool = False
+ enable_sstable_data_integrity_check: bool = False
+ enable_sstable_key_validation: bool = None
+ cpu_scheduler: bool = True
+ view_building: bool = True
+ enable_sstables_mc_format: bool = True
+ enable_dangerous_direct_import_of_cassandra_counters: bool = False
+ enable_shard_aware_drivers: bool = True
+ enable_ipv6_dns_lookup: bool = False
+ abort_on_internal_error: bool = False
+ max_partition_key_restrictions_per_query: int = 100
+ max_clustering_key_restrictions_per_query: int = 100
+ max_memory_for_unlimited_query: int = 1048576
+ initial_sstable_loading_concurrency: str = '4u'
+ enable_3_1_0_compatibility_mode: bool = False
+ enable_user_defined_functions: bool = False
+ user_defined_function_time_limit_ms: int = 10
+ user_defined_function_allocation_limit_bytes: int = 1024 * 1024
+ user_defined_function_contiguous_allocation_limit_bytes: int = 1024 * 1024
+ alternator_port: int = 0
+ alternator_https_port: int = 0
+ alternator_address: str = "0.0.0.0"
+ alternator_enforce_authorization: bool = False
+ alternator_write_isolation: Literal["unsafe_rmw", "only_rmw_uses_lwt", "forbid_rmw", "always_use_lwt"] = None
+ alternator_streams_time_window_s: int = 10
+ abort_on_ebadf: bool = True
+ redis_port: int = 0
+ redis_ssl_port: int = 0
+ redis_read_consistency_level: str = "LOCAL_QUORUM"
+ redis_write_consistency_level: str = "LOCAL_QUORUM"
+ redis_database_count: int = 16
+ redis_keyspace_replication_strategy: Literal['SimpleStrategy', 'NetworkTopologyStrategy'] = 'SimpleStrategy'
+ default_log_level: str = None
+ logger_log_level: str = None
+ log_to_stdout: bool = None
+ log_to_syslog: bool = None
+ authenticator_user: str = None
+ authenticator_password: str = None
+ workdir: str = None
+
+ ldap_attr_role: str = None
+ ldap_bind_dn: str = None
+ ldap_bind_passwd: str = None
+ ldap_url_template: str = None
+ saslauthd_socket_path: str = None
+
+ def _update_dict(self, obj: dict, fields_data: dict):
+ for attr_name, attr_value in obj.items():
+ attr_info = fields_data.get(attr_name, None)
+ if attr_info is None:
+ raise ValueError("Provided unknown attribute `%s`" % (attr_name,))
+ if hasattr(attr_info.type, "__attrs_attrs__"):
+ if attr_value is not None:
+ if not isinstance(attr_value, dict):
+ raise ValueError("Unexpected data `%s` in attribute `%s`" % (
+ type(attr_value), attr_name))
+ attr_value = attr_info.type(**attr_value)
+ setattr(self, attr_name, attr_value)
+
+ def update(self, *objects: Union['ScyllaYaml', dict]):
+ """
+ Do the same as dict.update, with one exception.
+ It ignores whatever key if it's value equal to default
+ This comes from module `attr` and probably could be tackled there
+ """
+ fields_data = self.__fields__
+ for obj in objects:
+ if isinstance(obj, ScyllaYaml):
+ for attr_name, attr_value in obj.__dict__.items():
+ attr_type = fields_data.get(attr_name)
+ if attr_value != attr_type.default and attr_value != getattr(self, attr_name, None):
+ setattr(self, attr_name, attr_value)
+ elif isinstance(obj, dict):
+ self._update_dict(obj, fields_data=fields_data)
+ else:
+ raise ValueError("Only dict or ScyllaYaml is accepted")
+ return self
+
+ def __copy__(self):
+ copy_object = self.__class__()
+ copy_object.__setstate__(self.__getstate__()) # pylint: disable=no-member
+ return copy_object
+
+ copy = __copy__
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/PR-provision-test.result.json b/unit_tests/test_data/test_scylla_yaml_builders/PR-provision-test.result.json
--- a/unit_tests/test_data/test_scylla_yaml_builders/PR-provision-test.result.json
+++ b/unit_tests/test_data/test_scylla_yaml_builders/PR-provision-test.result.json
@@ -0,0 +1,18 @@
+{
+ "broadcast_address": "__NODE_PUBLIC_ADDRESS__",
+ "listen_address": "__NODE_PRIVATE_ADDRESS__",
+ "endpoint_snitch": "org.apache.cassandra.locator.Ec2MultiRegionSnitch",
+ "rpc_address": "__NODE_PRIVATE_ADDRESS__",
+ "seed_provider": [
+ {
+ "class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
+ "parameters": [
+ {
+ "seeds": "__SEED_NODE_IPS__"
+ }
+ ]
+ }
+ ],
+ "broadcast_rpc_address": "__NODE_PUBLIC_ADDRESS__",
+ "experimental": true
+}
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/PR-provision-test.yaml b/unit_tests/test_data/test_scylla_yaml_builders/PR-provision-test.yaml
--- a/unit_tests/test_data/test_scylla_yaml_builders/PR-provision-test.yaml
+++ b/unit_tests/test_data/test_scylla_yaml_builders/PR-provision-test.yaml
@@ -0,0 +1,33 @@
+test_duration: 60
+max_partitions_in_test_table: 10
+stress_cmd: ["cassandra-stress write cl=QUORUM duration=1m -schema 'replication(factor=3) compaction(strategy=SizeTieredCompactionStrategy)' -port jmx=6868 -mode cql3 native -rate threads=1000 -pop seq=1..10000000 -log interval=5",
+ "cassandra-stress counter_write cl=QUORUM duration=1m -schema 'replication(factor=3) compaction(strategy=DateTieredCompactionStrategy)' -port jmx=6868 -mode cql3 native -rate threads=5 -pop seq=1..10000000"
+ ]
+
+prepare_write_cmd: ["scylla-bench -workload=sequential -mode=write -max-rate=300 -replication-factor=3 -partition-count=10 -clustering-row-count=100 -clustering-row-size=5120 -concurrency=7 -rows-per-request=10",
+ "scylla-bench -workload=uniform -mode=read -replication-factor=3 -partition-count=10 -clustering-row-count=100 -clustering-row-size=5120 -rows-per-request=10 -concurrency=7 -max-rate=32000 -duration=1m"
+ ]
+
+n_loaders: 1
+instance_type_db: 'i3.large'
+n_monitor_nodes: 1
+n_db_nodes: 3
+system_auth_rf: 0
+
+nemesis_class_name: NonDisruptiveMonkey
+nemesis_interval: 1
+
+user_prefix: 'PR-provision-test'
+instance_provision: 'spot'
+
+gce_image_db: 'https://www.googleapis.com/compute/v1/projects/centos-cloud/global/images/family/centos-7'
+
+scylla_version: 4.4.1
+scylla_mgmt_repo: 'http://downloads.scylladb.com/rpm/centos/scylladb-manager-2.3.repo'
+scylla_mgmt_agent_repo: 'http://downloads.scylladb.com/rpm/centos/scylladb-manager-2.3.repo'
+
+workaround_kernel_bug_for_iotune: true
+
+post_behavior_db_nodes: "destroy"
+post_behavior_loader_nodes: "destroy"
+post_behavior_monitor_nodes: "destroy"
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/artifact_amazon2.result.json b/unit_tests/test_data/test_scylla_yaml_builders/artifact_amazon2.result.json
--- a/unit_tests/test_data/test_scylla_yaml_builders/artifact_amazon2.result.json
+++ b/unit_tests/test_data/test_scylla_yaml_builders/artifact_amazon2.result.json
@@ -0,0 +1,18 @@
+{
+ "broadcast_address": "__NODE_PUBLIC_ADDRESS__",
+ "listen_address": "__NODE_PRIVATE_ADDRESS__",
+ "endpoint_snitch": "org.apache.cassandra.locator.Ec2MultiRegionSnitch",
+ "rpc_address": "__NODE_PRIVATE_ADDRESS__",
+ "seed_provider": [
+ {
+ "class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
+ "parameters": [
+ {
+ "seeds": "__SEED_NODE_IPS__"
+ }
+ ]
+ }
+ ],
+ "broadcast_rpc_address": "__NODE_PUBLIC_ADDRESS__",
+ "experimental": true
+}
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/artifact_amazon2.yaml b/unit_tests/test_data/test_scylla_yaml_builders/artifact_amazon2.yaml
--- a/unit_tests/test_data/test_scylla_yaml_builders/artifact_amazon2.yaml
+++ b/unit_tests/test_data/test_scylla_yaml_builders/artifact_amazon2.yaml
@@ -0,0 +1,21 @@
+ami_db_scylla_user: 'ec2-user'
+ami_id_db_scylla: 'ami-099a8245f5daa82bf'
+aws_root_disk_size_db: 50
+backtrace_decoding: false
+cluster_backend: 'aws'
+instance_type_db: 'i3.large'
+instance_provision: "spot"
+instance_provision_fallback_on_demand: true
+logs_transport: 'ssh'
+n_db_nodes: 1
+n_loaders: 0
+n_monitor_nodes: 0
+nemesis_class_name: 'NoOpMonkey'
+region_name: 'eu-west-1'
+scylla_linux_distro: 'centos'
+scylla_repo: 'https://s3.amazonaws.com/downloads.scylladb.com/rpm/centos/scylla-3.3.repo'
+system_auth_rf: 1
+test_duration: 60
+use_preinstalled_scylla: false
+user_prefix: 'artifacts-amazon2'
+scylla_mgmt_agent_repo: 'http://downloads.scylladb.com.s3.amazonaws.com/manager/rpm/unstable/centos/master/latest/scylla-manager.repo'
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/cdc-replication-longevity.result.json b/unit_tests/test_data/test_scylla_yaml_builders/cdc-replication-longevity.result.json
--- a/unit_tests/test_data/test_scylla_yaml_builders/cdc-replication-longevity.result.json
+++ b/unit_tests/test_data/test_scylla_yaml_builders/cdc-replication-longevity.result.json
@@ -0,0 +1,18 @@
+{
+ "broadcast_address": "__NODE_PRIVATE_ADDRESS__",
+ "listen_address": "0.0.0.0",
+ "endpoint_snitch": "org.apache.cassandra.locator.Ec2MultiRegionSnitch",
+ "rpc_address": "0.0.0.0",
+ "seed_provider": [
+ {
+ "class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
+ "parameters": [
+ {
+ "seeds": "__SEED_NODE_IPS__"
+ }
+ ]
+ }
+ ],
+ "broadcast_rpc_address": "__NODE_PRIVATE_ADDRESS__",
+ "experimental": true
+}
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/cdc-replication-longevity.yaml b/unit_tests/test_data/test_scylla_yaml_builders/cdc-replication-longevity.yaml
--- a/unit_tests/test_data/test_scylla_yaml_builders/cdc-replication-longevity.yaml
+++ b/unit_tests/test_data/test_scylla_yaml_builders/cdc-replication-longevity.yaml
@@ -0,0 +1,27 @@
+test_duration: 900
+
+user_prefix: 'cdc-replication-longevity'
+db_type: mixed_scylla
+
+n_db_nodes: 3
+instance_type_db: 'i3.large'
+
+n_test_oracle_db_nodes: 1
+instance_type_db_oracle: 'i3.large'
+
+n_loaders: 1
+instance_type_loader: 'c5.large'
+
+n_monitor_nodes: 1
+
+nemesis_class_name: 'CategoricalMonkey'
+nemesis_interval: 5
+
+extra_network_interface: True
+
+gemini_cmd: "gemini --duration 30m --warmup 0s -c 4 -m write --non-interactive --cql-features basic --max-mutation-retries 100 --max-mutation-retries-backoff 100ms --replication-strategy \"{'class': 'SimpleStrategy', 'replication_factor': '3'}\" --table-options \"cdc = {'enabled': true, 'ttl': 0}\" --use-server-timestamps --test-host-selection-policy token-aware"
+
+gemini_version: 'latest'
+# Required by SCT, although not used:
+gemini_schema_url: 'https://s3.amazonaws.com/scylla-gemini/Binaries/schema.json'
+ip_ssh_connections: 'private'
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/functional.result.json b/unit_tests/test_data/test_scylla_yaml_builders/functional.result.json
--- a/unit_tests/test_data/test_scylla_yaml_builders/functional.result.json
+++ b/unit_tests/test_data/test_scylla_yaml_builders/functional.result.json
@@ -0,0 +1,18 @@
+{
+ "broadcast_address": "__NODE_PUBLIC_ADDRESS__",
+ "listen_address": "__NODE_PRIVATE_ADDRESS__",
+ "endpoint_snitch": "org.apache.cassandra.locator.Ec2MultiRegionSnitch",
+ "rpc_address": "__NODE_PRIVATE_ADDRESS__",
+ "seed_provider": [
+ {
+ "class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
+ "parameters": [
+ {
+ "seeds": "__SEED_NODE_IPS__"
+ }
+ ]
+ }
+ ],
+ "broadcast_rpc_address": "__NODE_PUBLIC_ADDRESS__",
+ "experimental": true
+}
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/functional.yaml b/unit_tests/test_data/test_scylla_yaml_builders/functional.yaml
--- a/unit_tests/test_data/test_scylla_yaml_builders/functional.yaml
+++ b/unit_tests/test_data/test_scylla_yaml_builders/functional.yaml
@@ -0,0 +1,8 @@
+test_duration: 300
+k8s_deploy_monitoring: false
+use_mgmt: true
+user_prefix: 'functional'
+
+n_db_nodes: 3
+n_loaders: 0
+n_monitor_nodes: 0
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/jepsen.result.json b/unit_tests/test_data/test_scylla_yaml_builders/jepsen.result.json
--- a/unit_tests/test_data/test_scylla_yaml_builders/jepsen.result.json
+++ b/unit_tests/test_data/test_scylla_yaml_builders/jepsen.result.json
@@ -0,0 +1,18 @@
+{
+ "broadcast_address": "__NODE_PUBLIC_ADDRESS__",
+ "listen_address": "__NODE_PRIVATE_ADDRESS__",
+ "endpoint_snitch": "org.apache.cassandra.locator.Ec2MultiRegionSnitch",
+ "rpc_address": "__NODE_PRIVATE_ADDRESS__",
+ "seed_provider": [
+ {
+ "class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
+ "parameters": [
+ {
+ "seeds": "__SEED_NODE_IPS__"
+ }
+ ]
+ }
+ ],
+ "broadcast_rpc_address": "__NODE_PUBLIC_ADDRESS__",
+ "experimental": true
+}
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/jepsen.yaml b/unit_tests/test_data/test_scylla_yaml_builders/jepsen.yaml
--- a/unit_tests/test_data/test_scylla_yaml_builders/jepsen.yaml
+++ b/unit_tests/test_data/test_scylla_yaml_builders/jepsen.yaml
@@ -0,0 +1,20 @@
+test_duration: 720
+cluster_backend: 'gce'
+gce_image_username: 'sct'
+gce_image_db: 'https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/family/debian-10'
+gce_image_loader: 'https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/family/debian-10'
+gce_instance_type_db: 'n1-standard-4'
+gce_instance_type_loader: 'n1-highmem-8'
+gce_root_disk_type_db: 'pd-ssd'
+gce_root_disk_size_db: 50
+gce_n_local_ssd_disk_db: 1
+n_db_nodes: 5
+n_loaders: 1
+n_monitor_nodes: 1
+nemesis_class_name: 'NoOpMonkey'
+scylla_linux_distro: 'debian-buster'
+user_prefix: 'jepsen'
+use_legacy_cluster_init: true
+jepsen_scylla_repo: 'https://github.com/scylladb/jepsen.git'
+jepsen_test_count: 5
+jepsen_test_run_policy: any
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/longevity-10gb-3h.result.json b/unit_tests/test_data/test_scylla_yaml_builders/longevity-10gb-3h.result.json
--- a/unit_tests/test_data/test_scylla_yaml_builders/longevity-10gb-3h.result.json
+++ b/unit_tests/test_data/test_scylla_yaml_builders/longevity-10gb-3h.result.json
@@ -0,0 +1,20 @@
+{
+ "broadcast_address": "__NODE_IPV6_ADDRESS__",
+ "listen_address": "__NODE_IPV6_ADDRESS__",
+ "endpoint_snitch": "org.apache.cassandra.locator.Ec2MultiRegionSnitch",
+ "rpc_address": "__NODE_IPV6_ADDRESS__",
+ "seed_provider": [
+ {
+ "class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
+ "parameters": [
+ {
+ "seeds": "__SEED_NODE_IPS__"
+ }
+ ]
+ }
+ ],
+ "broadcast_rpc_address": "__NODE_IPV6_ADDRESS__",
+ "experimental": true,
+ "prometheus_address": "__NODE_IPV6_ADDRESS__",
+ "enable_ipv6_dns_lookup": true
+}
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/longevity-10gb-3h.yaml b/unit_tests/test_data/test_scylla_yaml_builders/longevity-10gb-3h.yaml
--- a/unit_tests/test_data/test_scylla_yaml_builders/longevity-10gb-3h.yaml
+++ b/unit_tests/test_data/test_scylla_yaml_builders/longevity-10gb-3h.yaml
@@ -0,0 +1,24 @@
+test_duration: 240
+stress_cmd: ["cassandra-stress write cl=QUORUM duration=180m -schema 'replication(factor=3) compaction(strategy=SizeTieredCompactionStrategy)' -port jmx=6868 -mode cql3 native -rate threads=1000 -pop seq=1..10000000 -log interval=5"
+ ]
+
+n_db_nodes: 6
+n_loaders: 2
+n_monitor_nodes: 1
+
+instance_type_db: 'i3.4xlarge'
+gce_instance_type_db: 'n1-highmem-16'
+gce_instance_type_loader: 'e2-standard-4'
+scylla_repo_loader: 'https://s3.amazonaws.com/downloads.scylladb.com/rpm/centos/scylla-4.3.repo'
+
+nemesis_class_name: 'SisyphusMonkey'
+nemesis_seed: '111'
+nemesis_interval: 2
+ssh_transport: 'libssh2'
+
+user_prefix: 'longevity-10gb-3h'
+space_node_threshold: 64424
+
+gce_n_local_ssd_disk_db: 2
+use_preinstalled_scylla: true
+ip_ssh_connections: 'ipv6'
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/manager-backup-1TB-gce.result.json b/unit_tests/test_data/test_scylla_yaml_builders/manager-backup-1TB-gce.result.json
--- a/unit_tests/test_data/test_scylla_yaml_builders/manager-backup-1TB-gce.result.json
+++ b/unit_tests/test_data/test_scylla_yaml_builders/manager-backup-1TB-gce.result.json
@@ -0,0 +1,18 @@
+{
+ "broadcast_address": "__NODE_PUBLIC_ADDRESS__",
+ "listen_address": "__NODE_PRIVATE_ADDRESS__",
+ "endpoint_snitch": "org.apache.cassandra.locator.Ec2MultiRegionSnitch",
+ "rpc_address": "__NODE_PRIVATE_ADDRESS__",
+ "seed_provider": [
+ {
+ "class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
+ "parameters": [
+ {
+ "seeds": "__SEED_NODE_IPS__"
+ }
+ ]
+ }
+ ],
+ "broadcast_rpc_address": "__NODE_PUBLIC_ADDRESS__",
+ "experimental": true
+}
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/manager-backup-1TB-gce.yaml b/unit_tests/test_data/test_scylla_yaml_builders/manager-backup-1TB-gce.yaml
--- a/unit_tests/test_data/test_scylla_yaml_builders/manager-backup-1TB-gce.yaml
+++ b/unit_tests/test_data/test_scylla_yaml_builders/manager-backup-1TB-gce.yaml
@@ -0,0 +1,40 @@
+test_duration: 600
+
+user_prefix: 'mgr-backup-1tb'
+cluster_backend: 'gce'
+
+prepare_write_cmd: ["cassandra-stress write cl=QUORUM n=550100150 -schema 'replication(factor=3)' -port jmx=6868 -mode cql3 native connectionsPerHost=64 -rate threads=150 throttle=45000/s -col 'size=FIXED(200) n=FIXED(5)' -pop seq=1..550100150",
+ "cassandra-stress write cl=QUORUM n=550100150 -schema 'replication(factor=3)' -port jmx=6868 -mode cql3 native connectionsPerHost=64 -rate threads=150 throttle=45000/s -col 'size=FIXED(200) n=FIXED(5)' -pop seq=550100151..1100200300",
+ ]
+stress_read_cmd: ["cassandra-stress read cl=QUORUM duration=580m -port jmx=6868 -mode cql3 native connectionsPerHost=64 -rate threads=150 -pop seq=1..1100200300 -log interval=5 -col 'size=FIXED(200) n=FIXED(5)'",
+ ]
+round_robin: true
+
+pre_create_schema: true
+compaction_strategy: 'NullCompactionStrategy'
+post_prepare_cql_cmds: "ALTER TABLE keyspace1.standard1 WITH compaction = {'class': 'SizeTieredCompactionStrategy'}"
+prepare_wait_no_compactions_timeout: 120
+
+n_db_nodes: 4
+n_loaders: 2
+n_monitor_nodes: 1
+
+gce_instance_type_db: 'n1-highmem-16'
+gce_n_local_ssd_disk_db: 8
+gce_instance_type_loader: 'e2-standard-16'
+
+backup_bucket_location: 'manager-backup-tests-us-east1'
+nemesis_class_name: 'MgmtBackup'
+nemesis_interval: 30
+nemesis_filter_seeds: false
+
+gce_image_db: 'https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/family/ubuntu-1804-lts'
+scylla_linux_distro: 'ubuntu-bionic'
+scylla_mgmt_agent_repo: 'http://downloads.scylladb.com.s3.amazonaws.com/manager/deb/unstable/bionic/master/latest/scylladb-manager-master/scylla-manager.list'
+
+scylla_linux_distro_loader: 'centos'
+scylla_mgmt_repo: 'http://downloads.scylladb.com.s3.amazonaws.com/manager/rpm/unstable/centos/master/latest/scylla-manager.repo'
+
+space_node_threshold: 644245094
+
+nemesis_during_prepare: false
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/perf-regression.100threads.30M-keys.result.json b/unit_tests/test_data/test_scylla_yaml_builders/perf-regression.100threads.30M-keys.result.json
--- a/unit_tests/test_data/test_scylla_yaml_builders/perf-regression.100threads.30M-keys.result.json
+++ b/unit_tests/test_data/test_scylla_yaml_builders/perf-regression.100threads.30M-keys.result.json
@@ -0,0 +1,18 @@
+{
+ "broadcast_address": "__NODE_PUBLIC_ADDRESS__",
+ "listen_address": "__NODE_PRIVATE_ADDRESS__",
+ "endpoint_snitch": "org.apache.cassandra.locator.Ec2MultiRegionSnitch",
+ "rpc_address": "__NODE_PRIVATE_ADDRESS__",
+ "seed_provider": [
+ {
+ "class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
+ "parameters": [
+ {
+ "seeds": "__SEED_NODE_IPS__"
+ }
+ ]
+ }
+ ],
+ "broadcast_rpc_address": "__NODE_PUBLIC_ADDRESS__",
+ "experimental": true
+}
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/perf-regression.100threads.30M-keys.yaml b/unit_tests/test_data/test_scylla_yaml_builders/perf-regression.100threads.30M-keys.yaml
--- a/unit_tests/test_data/test_scylla_yaml_builders/perf-regression.100threads.30M-keys.yaml
+++ b/unit_tests/test_data/test_scylla_yaml_builders/perf-regression.100threads.30M-keys.yaml
@@ -0,0 +1,24 @@
+test_duration: 300
+
+stress_cmd_w: "cassandra-stress write no-warmup cl=QUORUM duration=60m -schema 'replication(factor=3)' -port jmx=6868 -mode cql3 native -rate threads=100 -pop seq=1..30000000"
+prepare_write_cmd: "cassandra-stress write no-warmup cl=QUORUM n=30000000 -schema 'replication(factor=3)' -port jmx=6868 -mode cql3 native -rate threads=100 -pop seq=1..30000000"
+stress_cmd_r: "cassandra-stress read no-warmup cl=QUORUM duration=50m -schema 'replication(factor=3)' -port jmx=6868 -mode cql3 native -rate threads=100 -pop 'dist=gauss(1..30000000,15000000,1500000)' "
+stress_cmd_m: "cassandra-stress mixed no-warmup cl=QUORUM duration=50m -schema 'replication(factor=3)' -port jmx=6868 -mode cql3 native -rate threads=100 -pop 'dist=gauss(1..30000000,15000000,1500000)' "
+
+n_db_nodes: 3
+n_loaders: 4
+n_monitor_nodes: 1
+
+instance_type_db: 'i3.2xlarge'
+instance_type_loader: 'c4.2xlarge'
+instance_type_monitor: 't3.small'
+
+user_prefix: 'perf-regression'
+space_node_threshold: 644245094
+
+backtrace_decoding: false
+
+store_perf_results: true
+use_mgmt: false
+send_email: true
+email_recipients: ['scylla-pe...@scylladb.com']
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/rolling-upgrade.result.json b/unit_tests/test_data/test_scylla_yaml_builders/rolling-upgrade.result.json
--- a/unit_tests/test_data/test_scylla_yaml_builders/rolling-upgrade.result.json
+++ b/unit_tests/test_data/test_scylla_yaml_builders/rolling-upgrade.result.json
@@ -0,0 +1,26 @@
+{
+ "broadcast_address": "__NODE_PUBLIC_ADDRESS__",
+ "listen_address": "__NODE_PRIVATE_ADDRESS__",
+ "endpoint_snitch": "org.apache.cassandra.locator.Ec2MultiRegionSnitch",
+ "rpc_address": "__NODE_PRIVATE_ADDRESS__",
+ "seed_provider": [
+ {
+ "class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
+ "parameters": [
+ {
+ "seeds": "__SEED_NODE_IPS__"
+ }
+ ]
+ }
+ ],
+ "broadcast_rpc_address": "__NODE_PUBLIC_ADDRESS__",
+ "authenticator": "org.apache.cassandra.auth.PasswordAuthenticator",
+ "server_encryption_options": {
+ "internode_encryption": "all",
+ "certificate": "/etc/scylla/ssl_conf/db.crt",
+ "keyfile": "/etc/scylla/ssl_conf/db.key",
+ "truststore": "/etc/scylla/ssl_conf/cadb.pem"
+ },
+ "experimental": true,
+ "saslauthd_socket_path": "/run/saslauthd/mux"
+}
diff --git a/unit_tests/test_data/test_scylla_yaml_builders/rolling-upgrade.yaml b/unit_tests/test_data/test_scylla_yaml_builders/rolling-upgrade.yaml
--- a/unit_tests/test_data/test_scylla_yaml_builders/rolling-upgrade.yaml
+++ b/unit_tests/test_data/test_scylla_yaml_builders/rolling-upgrade.yaml
@@ -0,0 +1,53 @@
+# TODO: need to qualify on GCE and AWS
+
+test_duration: 300
+
+# workloads
+write_stress_during_entire_test: cassandra-stress write no-warmup cl=QUORUM n=20100200 -schema 'keyspace=keyspace_entire_test replication(factor=3) compression=LZ4Compressor compaction(strategy=SizeTieredCompactionStrategy)' -port jmx=6868 -mode cql3 native compression=lz4 -rate threads=100 -pop seq=1..20100200 -log interval=5
+verify_stress_after_cluster_upgrade: cassandra-stress read no-warmup cl=QUORUM n=20100200 -schema 'keyspace=keyspace_entire_test replication(factor=3) compression=LZ4Compressor' -port jmx=6868 -mode cql3 native compression=lz4 -rate threads=1000 -pop seq=1..20100200 -log interval=5
+prepare_write_stress: cassandra-stress write no-warmup cl=QUORUM n=10100200 -schema 'replication(factor=3) compression=LZ4Compressor compaction(strategy=LeveledCompactionStrategy)' -port jmx=6868 -mode cql3 native compression=lz4 -rate threads=1000 -pop seq=1..10100200 -log interval=5
+stress_cmd_read_cl_quorum: cassandra-stress read no-warmup cl=QUORUM n=10100200 -schema 'replication(factor=3) compression=DeflateCompressor' -port jmx=6868 -mode cql3 native compression=none -rate threads=1000 -pop seq=1..10100200 -log interval=5
+stress_cmd_read_10m: cassandra-stress read no-warmup cl=QUORUM duration=10m -schema 'replication(factor=3) compression=SnappyCompressor' -port jmx=6868 -mode cql3 native compression=snappy -rate threads=1000 -pop seq=1..10100200 -log interval=5
+stress_cmd_read_60m: cassandra-stress read no-warmup cl=QUORUM duration=60m -schema 'replication(factor=3) compression=LZ4Compressor' -port jmx=6868 -mode cql3 native compression=lz4 -rate threads=1000 -pop seq=1..10100200 -log interval=5
+stress_cmd_complex_prepare: cassandra-stress user no-warmup profile=/tmp/complex_schema.yaml ops'(insert=1)' cl=ALL n=5000000 -mode cql3 native -rate threads=1000 -pop seq=1..5000000
+stress_cmd_complex_verify_read: cassandra-stress user no-warmup profile=/tmp/complex_schema.yaml ops'(read1=1,read3=1)' cl=ONE n=5000000 -mode cql3 native -rate threads=1000 -pop seq=1..5000000
+stress_cmd_complex_verify_more: cassandra-stress user no-warmup profile=/tmp/complex_schema.yaml ops'(read1=1,read2=1,read3=1,update_static=1,update_ttl=1,update_diff1_ts=1,update_diff2_ts=1,update_same1_ts=1,update_same2_ts=1)' cl=ALL n=5000000 -mode cql3 native -rate threads=200 -pop seq=1..5000000
+stress_cmd_complex_verify_delete: cassandra-stress user no-warmup profile=/tmp/complex_schema.yaml ops'(delete_row=1)' cl=ALL n=500000 -mode cql3 native -rate threads=200 -pop seq=1..500000
+
+n_db_nodes: 4
+n_loaders: 1
+n_monitor_nodes: 1
+
+instance_type_db: 'i3.2xlarge'
+
+user_prefix: 'rolling-upgrade'
+
+server_encrypt: true
+authenticator: 'PasswordAuthenticator'
+authenticator_user: 'cassandra'
+authenticator_password: 'cassandra'
+
+authorization_in_upgrade: 'CassandraAuthorizer'
+remove_authorization_in_rollback: true
+recover_system_tables: true
+
+append_scylla_args: '--blocked-reactor-notify-ms 500 --abort-on-lsa-bad-alloc 1 --abort-on-seastar-bad-alloc --abort-on-internal-error 1'
+
+# those are needed to be give by the job, via environment variable
+# for the base version
+# SCT_SCYLLA_VERSION=3.0 or SCT_SCYLLA_REPO=
+# for the upgrading version you need:
+# SCT_NEW_SCYLLA_REPO=https://s3.amazonaws.com/downloads.scylladb.com/enterprise/rpm/unstable/centos/e438e3e3ce41f6c878b111f5c19bf2b28aa51098-2bdfa9f7ef592edaf15e028faf3b7f695f39ebc1-e616363bdc0e7a0d193a157851d7c15d952a0aad-a6b2b2355c666b1893f702a587287da978aeec22/53/scylla.repo
+
+use_mgmt: false
+
+stress_cdclog_reader_cmd: "cdc-stressor -username cassandra -password cassandra -duration 2h -stream-query-round-duration 30s"
+
+gemini_cmd: "gemini -d --duration 2h \
+-c 10 -m write -f --non-interactive --cql-features normal \
+--max-mutation-retries 5 --max-mutation-retries-backoff 500ms \
+--async-objects-stabilization-attempts 5 --async-objects-stabilization-backoff 500ms \
+--replication-strategy \"{'class': 'SimpleStrategy', 'replication_factor': '3'}\" \
+--table-options \"cdc={'enabled': true}\" --test-username cassandra --test-password cassandra"
+gemini_version: 'latest'
+gemini_schema_url: 'https://s3.amazonaws.com/scylla-gemini/Binaries/schema.json' # currently is not used
diff --git a/unit_tests/test_scylla_yaml.py b/unit_tests/test_scylla_yaml.py
--- a/unit_tests/test_scylla_yaml.py
+++ b/unit_tests/test_scylla_yaml.py
@@ -0,0 +1,422 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
+
+import unittest
+
+from sdcm.provision.scylla_yaml import ServerEncryptionOptions, ClientEncryptionOptions, SeedProvider, ScyllaYaml
+from sdcm.provision.scylla_yaml.auxiliaries import RequestSchedulerOptions
+
+
+class ScyllaYamlTest(unittest.TestCase):
+
+ @staticmethod
+ def _run_test(object_type, init_params: dict, expected_without_defaults: dict, expected_with_defaults: dict):
+ instance = object_type(**init_params)
+ assert instance.dict(exclude_unset=True) == expected_without_defaults
+ assert instance.dict(exclude_unset=False) == expected_with_defaults
+
+ def test_server_encryption_options(self):
+ self._run_test(
+ object_type=ServerEncryptionOptions,
+ init_params={
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key',
+ },
+ expected_without_defaults={
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key',
+ },
+ expected_with_defaults={
+ 'internode_encryption': 'none',
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key',
+ 'truststore': None,
+ 'priority_string': None,
+ 'require_client_auth': False
+ }
+ )
+
+ def test_client_encryption_options(self):
+ self._run_test(
+ object_type=ClientEncryptionOptions,
+ init_params={
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key',
+ 'truststore': '/tmp/123.pem',
+ 'require_client_auth': True,
+ },
+ expected_without_defaults={
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key',
+ 'truststore': '/tmp/123.pem',
+ 'require_client_auth': True
+ },
+ expected_with_defaults={
+ 'enabled': False,
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key',
+ 'truststore': '/tmp/123.pem',
+ 'priority_string': None,
+ 'require_client_auth': True
+ }
+ )
+
+ def test_request_scheduler_options(self):
+ self._run_test(
+ object_type=RequestSchedulerOptions,
+ init_params={
+ 'throttle_limit': 100,
+ 'default_weight': 1,
+ },
+ expected_without_defaults={'throttle_limit': 100, 'default_weight': 1},
+ expected_with_defaults={'throttle_limit': 100, 'default_weight': 1, 'weights': 1}
+ )
+
+ def test_seed_provider(self):
+ self._run_test(
+ object_type=SeedProvider,
+ init_params={
+ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'parameters': [{'seeds': ['1.1.1.1', '2.2.2.2']}]
+ },
+ expected_without_defaults={
+ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'parameters': [{'seeds': ['1.1.1.1', '2.2.2.2']}]},
+ expected_with_defaults={
+ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'parameters': [{'seeds': ['1.1.1.1', '2.2.2.2']}]
+ }
+ )
+
+ def test_scylla_yaml(self):
+ self._run_test(
+ object_type=ScyllaYaml,
+ init_params={
+ 'log_to_stdout': True,
+ 'auto_adjust_flush_quota': False,
+ 'background_writer_scheduling_quota': 1.0,
+ 'listen_address': "localhost",
+ 'prometheus_prefix': 'someprefix',
+ 'server_encryption_options': ServerEncryptionOptions(
+ internode_encryption='none',
+ certificate='/tmp/123.crt',
+ keyfile='/tmp/123.key',
+ ),
+ 'client_encryption_options': ClientEncryptionOptions(
+ enabled=False,
+ certificate='/tmp/123.crt',
+ keyfile='/tmp/123.key',
+ require_client_auth=True,
+ ),
+ 'seed_provider': [
+ SeedProvider(
+ class_name='org.apache.cassandra.locator.SimpleSeedProvider',
+ parameters=[{'seeds': ['1.1.1.1', '2.2.2.2']}]),
+ ],
+ },
+ expected_without_defaults={
+ 'background_writer_scheduling_quota': 1.0,
+ 'auto_adjust_flush_quota': False,
+ 'listen_address': 'localhost',
+ 'seed_provider': [
+ {
+ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'parameters': [{'seeds': ['1.1.1.1', '2.2.2.2']}]
+ }
+ ],
+ 'server_encryption_options': {
+ 'internode_encryption': 'none',
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key'
+ },
+ 'client_encryption_options': {
+ 'enabled': False,
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key',
+ 'require_client_auth': True
+ },
+ 'prometheus_prefix': 'someprefix',
+ 'log_to_stdout': True,
+ },
+ expected_with_defaults={
+ 'broadcast_address': '',
+ 'api_port': 10000,
+ 'api_address': '',
+ 'ssl_storage_port': 7001,
+ 'background_writer_scheduling_quota': 1.0,
+ 'auto_adjust_flush_quota': False,
+ 'memtable_flush_static_shares': 0,
+ 'compaction_static_shares': 0,
+ 'compaction_enforce_min_threshold': False,
+ 'cluster_name': '',
+ 'listen_address': 'localhost',
+ 'listen_interface': 'eth0',
+ 'listen_interface_prefer_ipv6': False,
+ 'commitlog_directory': '',
+ 'data_file_directories': None,
+ 'hints_directory': '',
+ 'view_hints_directory': '',
+ 'saved_caches_directory': '',
+ 'commit_failure_policy': 'stop',
+ 'disk_failure_policy': 'stop',
+ 'endpoint_snitch': 'org.apache.cassandra.locator.SimpleSnitch',
+ 'rpc_address': 'localhost',
+ 'rpc_interface': 'eth1',
+ 'rpc_interface_prefer_ipv6': False,
+ 'seed_provider': [
+ {
+ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'parameters': [{'seeds': ['1.1.1.1', '2.2.2.2']}],
+ },
+ ],
+ 'compaction_throughput_mb_per_sec': 16,
+ 'compaction_large_partition_warning_threshold_mb': 1000,
+ 'compaction_large_row_warning_threshold_mb': 10,
+ 'compaction_large_cell_warning_threshold_mb': 1,
+ 'compaction_rows_count_warning_threshold': 100000,
+ 'memtable_total_space_in_mb': 0,
+ 'concurrent_reads': 32,
+ 'concurrent_writes': 32,
+ 'concurrent_counter_writes': 32,
+ 'incremental_backups': False,
+ 'snapshot_before_compaction': False,
+ 'phi_convict_threshold': 8,
+ 'commitlog_sync': 'periodic',
+ 'commitlog_segment_size_in_mb': 64,
+ 'commitlog_sync_period_in_ms': 10000,
+ 'commitlog_sync_batch_window_in_ms': 10000,
+ 'commitlog_total_space_in_mb': -1,
+ 'commitlog_reuse_segments': True,
+ 'commitlog_use_o_dsync': True,
+ 'compaction_preheat_key_cache': True,
+ 'concurrent_compactors': 0,
+ 'in_memory_compaction_limit_in_mb': 64,
+ 'preheat_kernel_page_cache': False,
+ 'sstable_preemptive_open_interval_in_mb': 50,
+ 'defragment_memory_on_idle': False,
+ 'memtable_allocation_type': 'heap_buffers',
+ 'memtable_cleanup_threshold': 0.11,
+ 'file_cache_size_in_mb': 512,
+ 'memtable_flush_queue_size': 4,
+ 'memtable_flush_writers': 1,
+ 'memtable_heap_space_in_mb': 0,
+ 'memtable_offheap_space_in_mb': 0,
+ 'column_index_size_in_kb': 64,
+ 'index_summary_capacity_in_mb': 0,
+ 'index_summary_resize_interval_in_minutes': 60,
+ 'reduce_cache_capacity_to': 0.6,
+ 'reduce_cache_sizes_at': 0.85,
+ 'stream_throughput_outbound_megabits_per_sec': 400,
+ 'inter_dc_stream_throughput_outbound_megabits_per_sec': 0,
+ 'trickle_fsync': False,
+ 'trickle_fsync_interval_in_kb': 10240,
+ 'auto_bootstrap': True,
+ 'batch_size_warn_threshold_in_kb': 5,
+ 'batch_size_fail_threshold_in_kb': 50,
+ 'listen_on_broadcast_address': False,
+ 'initial_token': None,
+ 'num_tokens': 1,
+ 'partitioner': 'org.apache.cassandra.dht.Murmur3Partitioner',
+ 'storage_port': 7000,
+ 'auto_snapshot': True,
+ 'key_cache_keys_to_save': 0,
+ 'key_cache_save_period': 14400,
+ 'key_cache_size_in_mb': 100,
+ 'row_cache_keys_to_save': 0,
+ 'row_cache_size_in_mb': 0,
+ 'row_cache_save_period': 0,
+ 'memory_allocator': 'NativeAllocator',
+ 'counter_cache_size_in_mb': 0,
+ 'counter_cache_save_period': 7200,
+ 'counter_cache_keys_to_save': 0,
+ 'tombstone_warn_threshold': 1000,
+ 'tombstone_failure_threshold': 100000,
+ 'range_request_timeout_in_ms': 10000,
+ 'read_request_timeout_in_ms': 5000,
+ 'counter_write_request_timeout_in_ms': 5000,
+ 'cas_contention_timeout_in_ms': 1000,
+ 'truncate_request_timeout_in_ms': 60000,
+ 'write_request_timeout_in_ms': 2000,
+ 'request_timeout_in_ms': 10000,
+ 'cross_node_timeout': False,
+ 'internode_send_buff_size_in_bytes': 0,
+ 'internode_recv_buff_size_in_bytes': 0,
+ 'internode_compression': 'none',
+ 'inter_dc_tcp_nodelay': False,
+ 'streaming_socket_timeout_in_ms': 0,
+ 'start_native_transport': True,
+ 'native_transport_port': 9042,
+ 'native_transport_port_ssl': 9142,
+ 'native_transport_max_threads': 128,
+ 'native_transport_max_frame_size_in_mb': 256,
+ 'native_shard_aware_transport_port': 19042,
+ 'native_shard_aware_transport_port_ssl': 19142,
+ 'broadcast_rpc_address': None,
+ 'rpc_port': 9160,
+ 'start_rpc': True,
+ 'rpc_keepalive': True,
+ 'rpc_max_threads': 0, 'rpc_min_threads': 16,
+ 'rpc_recv_buff_size_in_bytes': 0,
+ 'rpc_send_buff_size_in_bytes': 0,
+ 'rpc_server_type': 'sync',
+ 'cache_hit_rate_read_balancing': True,
+ 'dynamic_snitch_badness_threshold': 0,
+ 'dynamic_snitch_reset_interval_in_ms': 60000,
+ 'dynamic_snitch_update_interval_in_ms': 100,
+ 'hinted_handoff_enabled': 'enabled',
+ 'hinted_handoff_throttle_in_kb': 1024,
+ 'max_hint_window_in_ms': 10800000,
+ 'max_hints_delivery_threads': 2,
+ 'batchlog_replay_throttle_in_kb': 1024,
+ 'request_scheduler': 'org.apache.cassandra.scheduler.NoScheduler',
+ 'request_scheduler_id': None,
+ 'request_scheduler_options': None,
+ 'thrift_framed_transport_size_in_mb': 15,
+ 'thrift_max_message_length_in_mb': 16,
+ 'authenticator': 'org.apache.cassandra.auth.AllowAllAuthenticator',
+ 'internode_authenticator': 'disabled',
+ 'authorizer': 'org.apache.cassandra.auth.AllowAllAuthorizer',
+ 'role_manager': 'org.apache.cassandra.auth.CassandraRoleManager',
+ 'permissions_validity_in_ms': 10000,
+ 'permissions_update_interval_in_ms': 2000,
+ 'permissions_cache_max_entries': 1000,
+ 'server_encryption_options': {
+ 'internode_encryption': 'none',
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key',
+ 'truststore': None,
+ 'priority_string': None,
+ 'require_client_auth': False
+ },
+ 'client_encryption_options': {
+ 'enabled': False,
+ 'certificate': '/tmp/123.crt',
+ 'keyfile': '/tmp/123.key',
+ 'truststore': None,
+ 'priority_string': None,
+ 'require_client_auth': True
+ },
+ 'enable_in_memory_data_store': False,
+ 'enable_cache': True,
+ 'enable_commitlog': True,
+ 'volatile_system_keyspace_for_testing': False,
+ 'api_ui_dir': 'swagger-ui/dist/',
+ 'api_doc_dir': 'api/api-doc/',
+ 'load_balance': 'none',
+ 'consistent_rangemovement': True,
+ 'join_ring': True,
+ 'load_ring_state': True,
+ 'replace_node': '',
+ 'replace_token': '',
+ 'replace_address': '',
+ 'replace_address_first_boot': '',
+ 'override_decommission': False,
+ 'enable_repair_based_node_ops': True,
+ 'ring_delay_ms': 30000,
+ 'shadow_round_ms': 300000,
+ 'fd_max_interval_ms': 2000,
+ 'fd_initial_value_ms': 2000,
+ 'shutdown_announce_in_ms': 2000,
+ 'developer_mode': False,
+ 'skip_wait_for_gossip_to_settle': -1,
+ 'force_gossip_generation': -1,
+ 'experimental': False,
+ 'experimental_features': {},
+ 'lsa_reclamation_step': 1,
+ 'prometheus_port': 9180,
+ 'prometheus_address': '0.0.0.0',
+ 'prometheus_prefix': 'someprefix',
+ 'abort_on_lsa_bad_alloc': False,
+ 'alternator_address': '0.0.0.0',
+ 'murmur3_partitioner_ignore_msb_bits': 12,
+ 'virtual_dirty_soft_limit': 0.6,
+ 'sstable_summary_ratio': 0.0005,
+ 'large_memory_allocation_warning_threshold': 1048576,
+ 'enable_deprecated_partitioners': False,
+ 'enable_keyspace_column_family_metrics': False,
+ 'enable_sstable_data_integrity_check': False,
+ 'enable_sstable_key_validation': None,
+ 'cpu_scheduler': True,
+ 'view_building': True,
+ 'enable_sstables_mc_format': True,
+ 'enable_dangerous_direct_import_of_cassandra_counters': False,
+ 'enable_shard_aware_drivers': True,
+ 'enable_ipv6_dns_lookup': False,
+ 'abort_on_internal_error': False,
+ 'max_partition_key_restrictions_per_query': 100,
+ 'max_clustering_key_restrictions_per_query': 100,
+ 'max_memory_for_unlimited_query': 1048576,
+ 'initial_sstable_loading_concurrency': '4u',
+ 'enable_3_1_0_compatibility_mode': False,
+ 'enable_user_defined_functions': False,
+ 'user_defined_function_time_limit_ms': 10,
+ 'user_defined_function_allocation_limit_bytes': 1048576,
+ 'user_defined_function_contiguous_allocation_limit_bytes': 1048576,
+ 'alternator_port': 0,
+ 'alternator_https_port': 0,
+ 'alternator_enforce_authorization': False,
+ 'alternator_write_isolation': None,
+ 'alternator_streams_time_window_s': 10,
+ 'abort_on_ebadf': True,
+ 'redis_port': 0,
+ 'redis_ssl_port': 0,
+ 'redis_read_consistency_level': 'LOCAL_QUORUM',
+ 'redis_write_consistency_level': 'LOCAL_QUORUM',
+ 'redis_database_count': 16,
+ 'redis_keyspace_replication_strategy': 'SimpleStrategy',
+ 'default_log_level': None,
+ 'logger_log_level': None,
+ 'log_to_stdout': True,
+ 'log_to_syslog': None,
+ 'authenticator_user': None,
+ 'authenticator_password': None,
+ 'workdir': None,
+ 'ldap_attr_role': None,
+ 'ldap_bind_dn': None,
+ 'ldap_bind_passwd': None,
+ 'ldap_url_template': None,
+ 'saslauthd_socket_path': None
+ }
+ )
+
+ @staticmethod
+ def test_update():
+ yaml1 = ScyllaYaml(cluster_name='cluster1', redis_keyspace_replication_strategy='NetworkTopologyStrategy')
+ yaml2 = ScyllaYaml(
+ redis_keyspace_replication_strategy='SimpleStrategy',
+ client_encryption_options=ClientEncryptionOptions(
+ enabled=True,
+ certificate='/tmp/123.crt',
+ keyfile='/tmp/123.key',
+ truststore='/tmp/trust.pem',
+ ),
+ server_encryption_options=ServerEncryptionOptions(
+ internode_encryption='all',
+ certificate='/tmp/123.crt',
+ keyfile='/tmp/123.key',
+ truststore='/tmp/trust.pem',
+ )
+ )
+ yaml3 = ScyllaYaml(client_encryption_options=ClientEncryptionOptions())
+ yaml1.update(yaml2, yaml3)
+ assert yaml1 == ScyllaYaml(
+ cluster_name='cluster1',
+ # redis_keyspace_replication_strategy property is not getting changed because of the problem with update()
+ # which does not allow to make distinction between default value and value that equals to default
+ redis_keyspace_replication_strategy='NetworkTopologyStrategy',
+ server_encryption_options=ServerEncryptionOptions(internode_encryption='all',
+ certificate='/tmp/123.crt', keyfile='/tmp/123.key',
+ truststore='/tmp/trust.pem'),
+ client_encryption_options=ClientEncryptionOptions()
+ )
diff --git a/unit_tests/test_scylla_yaml_builders.py b/unit_tests/test_scylla_yaml_builders.py
--- a/unit_tests/test_scylla_yaml_builders.py
+++ b/unit_tests/test_scylla_yaml_builders.py
@@ -0,0 +1,571 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2021 ScyllaDB
+
+import contextlib
+import json
+import os
+import tempfile
+import unittest
+from typing import List
+from unittest.mock import patch
+
+from parameterized import parameterized
+
+from sdcm.cluster import BaseNode, BaseCluster, BaseScyllaCluster
+from sdcm.provision.common.utils import is_builtin
+from sdcm.provision.scylla_yaml import ScyllaYamlNodeAttrBuilder, ScyllaYamlClusterAttrBuilder, ScyllaYaml
+from sdcm.provision.scylla_yaml.auxiliaries import ScyllaYamlAttrBuilderBase
+from sdcm.sct_config import SCTConfiguration
+from sdcm.test_config import TestConfig
+
+from sdcm.utils.distro import Distro
+
+
+# pylint: disable=too-few-public-methods
+class FakeCluster:
+ seed_nodes_ips = ['1.1.1.1']
+
+
+# pylint: disable=too-few-public-methods
+class TestConfigWithLdap:
+ LDAP_ADDRESS = '1.1.1.1', '389'
+
+
+# pylint: disable=too-few-public-methods
+class TestConfigWithoutLdap:
+ LDAP_ADDRESS = None
+
+
+# pylint: disable=too-few-public-methods
+class FakeNode:
+ parent_cluster = FakeCluster()
+ public_ip_address = '2.2.2.2'
+ private_ip_address = '1.1.1.1'
+
+
+class ScyllaYamlClusterAttrBuilderBase(unittest.TestCase):
+ builder_class: ScyllaYamlAttrBuilderBase
+
+ def _run_test(self, builder_params: dict, expected_as_dict: dict = None):
+ instance = self.builder_class(**builder_params)
+ resulted_as_dict = instance.dict(exclude_defaults=True, exclude_unset=True)
+ if expected_as_dict:
+ assert resulted_as_dict == expected_as_dict
+ ScyllaYaml(**resulted_as_dict)
+
+
+class ScyllaYamlClusterAttrBuilderTest(ScyllaYamlClusterAttrBuilderBase):
+ builder_class = ScyllaYamlClusterAttrBuilder
+
+ def test_aws_single_noldap(self):
+ self._run_test(
+ builder_params={
+ 'node': FakeNode(),
+ 'params': {
+ 'cluster_backend': 'aws',
+ 'region_name': 'eu-west-1',
+ 'intra_node_comm_public': False,
+ 'authenticator': None,
+ 'authorizer': None,
+ 'alternator_port': None,
+ 'use_ldap_authorization': None,
+ 'use_ms_ad_ldap': None,
+ 'internode_encryption': False,
+ 'client_encrypt': False,
+ 'hinted_handoff': None,
+ }},
+ expected_as_dict={
+ 'alternator_enforce_authorization': False,
+ 'enable_ipv6_dns_lookup': False,
+ }
+ )
+
+ def test_aws_multi_openldap(self):
+ self._run_test(
+ builder_params={
+ 'test_config': TestConfigWithLdap,
+ 'params': {
+ 'cluster_backend': 'aws',
+ 'region_name': 'eu-west-1 eu-west-2',
+ 'intra_node_comm_public': False,
+ 'authenticator': 'com.scylladb.auth.SaslauthdAuthenticator',
+ 'authorizer': 'AllowAllAuthorizer',
+ 'alternator_port': None,
+ 'use_ldap_authorization': True,
+ 'use_ms_ad_ldap': None,
+ 'internode_encryption': False,
+ 'client_encrypt': False,
+ 'hinted_handoff': None,
+ }
+ },
+ expected_as_dict={
+ 'alternator_enforce_authorization': False,
+ 'authenticator': 'com.scylladb.auth.SaslauthdAuthenticator',
+ 'authorizer': 'AllowAllAuthorizer',
+ 'enable_ipv6_dns_lookup': False,
+ 'endpoint_snitch': 'org.apache.cassandra.locator.Ec2MultiRegionSnitch',
+ 'ldap_attr_role': 'cn',
+ 'ldap_bind_dn': 'cn=admin,dc=scylla-qa,dc=com', 'ldap_bind_passwd': 'scylla-0',
+ 'ldap_url_template': 'ldap://127.0.0.1:5001/dc=scylla-qa,dc=com?cn?sub?'
+ '(uniqueMember=uid={USER},ou=Person,dc=scylla-qa,dc=com)',
+ 'saslauthd_socket_path': '/run/saslauthd/mux'
+ },
+ )
+
+ def test_gce_single_openldap(self):
+ self._run_test(
+ builder_params={
+ 'test_config': TestConfigWithLdap,
+ 'params': {
+ 'cluster_backend': 'gce',
+ 'gce_datacenter': 'eu-west-1',
+ 'intra_node_comm_public': False,
+ 'authenticator': 'com.scylladb.auth.SaslauthdAuthenticator',
+ 'authorizer': 'CassandraAuthorizer',
+ 'alternator_port': True,
+ 'use_ldap_authorization': True,
+ 'use_ms_ad_ldap': False,
+ 'internode_encryption': False,
+ 'client_encrypt': False,
+ 'hinted_handoff': None,
+ }
+ },
+ expected_as_dict={
+ 'alternator_enforce_authorization': False,
+ 'alternator_port': True,
+ 'alternator_write_isolation': 'always_use_lwt',
+ 'authenticator': 'com.scylladb.auth.SaslauthdAuthenticator',
+ 'authorizer': 'CassandraAuthorizer',
+ 'enable_ipv6_dns_lookup': False,
+ 'ldap_attr_role': 'cn', 'ldap_bind_dn': 'cn=admin,dc=scylla-qa,dc=com',
+ 'ldap_bind_passwd': 'scylla-0',
+ 'ldap_url_template': 'ldap://1.1.1.1:389/dc=scylla-qa,dc=com?cn?sub?'
+ '(uniqueMember=uid={USER},ou=Person,dc=scylla-qa,dc=com)',
+ 'saslauthd_socket_path': '/run/saslauthd/mux'
+ }
+ )
+
+ def test_gce_multi_msldap(self):
+ self._run_test(
+ builder_params={
+ 'test_config': TestConfigWithLdap,
+ 'msldap_server_info': {
+ 'ldap_bind_dn': 'SOMEDN',
+ 'admin_password': 'PASSWORD',
+ 'server_address': '3.3.3.3'
+ },
+ 'params': {
+ 'cluster_backend': 'gce',
+ 'gce_datacenter': 'eu-west-1 eu-west-2',
+ 'intra_node_comm_public': False,
+ 'authenticator': 'com.scylladb.auth.SaslauthdAuthenticator',
+ 'authorizer': 'CassandraAuthorizer',
+ 'alternator_port': False,
+ 'use_ldap_authorization': True,
+ 'use_ms_ad_ldap': True,
+ 'internode_encryption': True,
+ 'client_encrypt': True,
+ 'hinted_handoff': True,
+ }
+ },
+ expected_as_dict={
+ 'alternator_enforce_authorization': False,
+ 'alternator_port': False,
+ 'authenticator': 'com.scylladb.auth.SaslauthdAuthenticator',
+ 'authorizer': 'CassandraAuthorizer',
+ 'enable_ipv6_dns_lookup': False,
+ 'endpoint_snitch': 'org.apache.cassandra.locator.GossipingPropertyFileSnitch',
+ 'ldap_attr_role': 'cn', 'ldap_bind_dn': 'SOMEDN', 'ldap_bind_passwd': 'PASSWORD',
+ 'ldap_url_template': 'ldap://3.3.3.3:389/dc=scylla-qa,dc=com?cn?sub?(member=CN={USER},dc=scylla-qa,dc=com)',
+ 'saslauthd_socket_path': '/run/saslauthd/mux',
+ }
+ )
+
+ def test_validation_gce(self):
+ with self.assertRaises(RuntimeError):
+ self._run_test(
+ builder_params={
+ 'test_config': TestConfigWithLdap,
+ 'params': {
+ 'cluster_backend': 'gce',
+ 'use_ldap_authorization': True,
+ 'use_ms_ad_ldap': True,
+ }
+ },
+ )
+
+ def test_validation_aws(self):
+ with self.assertRaises(RuntimeError):
+ self._run_test(
+ builder_params={
+ 'test_config': TestConfigWithoutLdap,
+ 'params': {
+ 'cluster_backend': 'aws',
+ 'use_ldap_authorization': True,
+ 'ip_ssh_connections': 'private',
+ }
+ },
+ )
+
+
+class ScyllaYamlNodeAttrBuilderTest(ScyllaYamlClusterAttrBuilderBase):
+ builder_class = ScyllaYamlNodeAttrBuilder
+
+ def test_aws_single_private_noextra(self):
+ self._run_test(
+ builder_params={
+ 'node': FakeNode(),
+ 'params': {
+ 'cluster_backend': 'aws',
+ 'region_name': 'eu-west-1',
+ 'ip_ssh_connections': 'private',
+ 'extra_network_interface': False,
+ 'intra_node_comm_public': False,
+ }
+ },
+ expected_as_dict={
+ 'listen_address': '1.1.1.1',
+ 'rpc_address': '1.1.1.1',
+ 'seed_provider': [
+ {
+ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'parameters': [{'seeds': '1.1.1.1'}]
+ }
+ ]
+ }
+ )
+
+ def test_aws_single_public_noextra(self):
+ self._run_test(
+ builder_params={
+ 'node': FakeNode(),
+ 'params': {
+ 'cluster_backend': 'aws',
+ 'region_name': 'eu-west-1',
+ 'ip_ssh_connections': 'private',
+ 'extra_network_interface': False,
+ 'intra_node_comm_public': True,
+ }
+ },
+ expected_as_dict={
+ 'broadcast_address': '2.2.2.2',
+ 'broadcast_rpc_address': '2.2.2.2',
+ 'listen_address': '1.1.1.1',
+ 'rpc_address': '1.1.1.1',
+ 'seed_provider': [
+ {
+ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'parameters': [{'seeds': '1.1.1.1'}]
+ }
+ ]
+ }
+ )
+
+ def test_aws_single_public_extra(self):
+ self._run_test(
+ builder_params={
+ 'node': FakeNode(),
+ 'params': {
+ 'cluster_backend': 'aws',
+ 'region_name': 'eu-west-1',
+ 'ip_ssh_connections': 'private',
+ 'extra_network_interface': True,
+ 'intra_node_comm_public': True,
+ }
+ },
+ expected_as_dict={
+ 'broadcast_address': '1.1.1.1',
+ 'broadcast_rpc_address': '1.1.1.1',
+ 'listen_address': '0.0.0.0',
+ 'rpc_address': '0.0.0.0',
+ 'seed_provider': [
+ {
+ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'parameters': [{'seeds': '1.1.1.1'}]
+ }
+ ]
+ }
+ )
+
+ def test_aws_multi_private_extra(self):
+ self._run_test(
+ builder_params={
+ 'node': FakeNode(),
+ 'params': {
+ 'cluster_backend': 'aws',
+ 'region_name': 'eu-west-1 eu-west-2',
+ 'ip_ssh_connections': 'private',
+ 'extra_network_interface': True,
+ 'intra_node_comm_public': False,
+ }
+ },
+ expected_as_dict={
+ 'broadcast_address': '1.1.1.1',
+ 'broadcast_rpc_address': '1.1.1.1',
+ 'listen_address': '0.0.0.0',
+ 'rpc_address': '0.0.0.0',
+ 'seed_provider': [
+ {
+ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
+ 'parameters': [{'seeds': '1.1.1.1'}]
+ }
+ ]
+ }
+ )
+
+ def test_negative_aws_single_public_noextra(self):
+ self._run_test(
+ builder_params={
+ 'node': FakeNode(),
+ 'params': {
+ 'cluster_backend': 'aws',
+ 'region_name': 'eu-west-1',
+ 'ip_ssh_connections': 'private',
+ 'extra_network_interface': False,
+ 'intra_node_comm_public': True,
+ }
+ },
+ )
+
+
+BASE_FOLDER = os.path.join(os.path.dirname(__file__), 'test_data/test_scylla_yaml_builders')
+
+
+class FakeRemoter: # pylint: disable=too-few-public-methods
+ def send_file(self, *_, **__):
+ pass
+
+ send_files = stop = start = run = sudo = send_file
+
+
+class ObjectDict(dict):
+ @staticmethod
+ def fix_name(name):
+ output = [name[0].lower()]
+ for char in name[1:]:
+ if char.isupper():
+ output.extend(['_', char.lower()])
+ else:
+ output.append(char)
+ return ''.join(output)
+
+ def __getattr__(self, item):
+ for key, value in self.items():
+ if self.fix_name(key) == item:
+ return value
+ raise AttributeError(f"There no {item} attribute")
+
+
+class DummyCluster(BaseScyllaCluster, BaseCluster): # pylint: disable=too-few-public-methods
+ nodes: List['DummyNode']
+
+ def __init__(self, params): # pylint: disable=super-init-not-called
+ self.nodes = []
+ self.params = params
+ self.name = 'dummy_cluster'
+
+ @property
+ def seed_nodes_ips(self):
+ return [node.ip_address for node in self.nodes]
+
+ def init_nodes(self):
+ for node in self.nodes:
+ self.node_config_setup(node, ','.join(self.seed_nodes_ips), self.get_endpoint_snitch())
+
+ # pylint: disable=too-many-arguments
+ def add_nodes(self, count, ec2_user_data='', dc_idx=0, rack=0, enable_auto_bootstrap=False):
+ pass
+
+
+class DummyNode(BaseNode): # pylint: disable=abstract-method
+ _system_log = None
+ is_enterprise = False
+ distro = Distro.CENTOS7
+
+ def __init__(self, node_num: int, parent_cluster, base_logdir: str):
+ self._private_ip_address = '1.1.1.' + str(node_num)
+ self._public_ip_address = '2.2.2.' + str(node_num)
+ self._ipv6_ip_address = 'aaaa:db8:aaaa:aaaa:aaaa:aaaa:aaaa:aaa' + str(node_num)
+ self._scylla_yaml = ScyllaYaml()
+ super().__init__(
+ base_logdir=base_logdir,
+ parent_cluster=parent_cluster,
+ name='node-' + str(node_num),
+ ssh_login_info=dict(key_file='~/.ssh/scylla-test'),
+ )
+
+ def _init_remoter(self, ssh_login_info):
+ self.remoter = FakeRemoter()
+
+ @contextlib.contextmanager
+ def remote_scylla_yaml(self):
+ yield self._scylla_yaml
+
+ def _get_private_ip_address(self):
+ return self._private_ip_address
+
+ def _get_public_ip_address(self):
+ return self._public_ip_address
+
+ def process_scylla_args(self, append_scylla_args=''):
+ pass
+
+ @property
+ def init_system(self):
+ return 'systemd'
+
+ def fix_scylla_server_systemd_config(self):
+ pass
+
+ def _get_ipv6_ip_address(self):
+ return self._ipv6_ip_address
+
+ def start_task_threads(self):
+ # disable all background threads
+ pass
+
+ def set_hostname(self):
+ pass
+
+ def set_keep_alive(self):
+ pass
+
+ @property
+ def system_log(self):
+ return self._system_log
+
+ @property
+ def is_nonroot_install(self): # pylint: disable=invalid-overridden-method
+ return False
+
+ @system_log.setter
+ def system_log(self, log):
+ self._system_log = log
+
+ def wait_ssh_up(self, verbose=True, timeout=500):
+ pass
+
+ def validate_scylla_yaml(self, expected_node_config, seed_node_ips):
+ with self.remote_scylla_yaml() as node_yaml:
+ expected_node_yaml = expected_node_config.replace(
+ '__NODE_PRIVATE_ADDRESS__', self.private_ip_address)
+ expected_node_yaml = expected_node_yaml.replace(
+ '__NODE_PUBLIC_ADDRESS__', self.public_ip_address)
+ expected_node_yaml = expected_node_yaml.replace(
+ '__SEED_NODE_IPS__', seed_node_ips)
+ expected_node_yaml = expected_node_yaml.replace('__NODE_IPV6_ADDRESS__', self.ipv6_ip_address)
+ assert json.loads(expected_node_yaml) == node_yaml.dict(exclude_unset=True, exclude_defaults=True)
+
+
+class IntegrationTests(unittest.TestCase):
+ get_scylla_ami_version_output = ObjectDict(**{
+ 'Architecture': 'x86_64', 'CreationDate': '2021-04-11T11:42:47.000Z',
+ 'ImageId': 'ami-01717260ff878ed92', 'ImageLocation': '797456418907/ScyllaDB 4.4.1',
+ 'ImageType': 'machine', 'Public': True, 'OwnerId': '797456418907',
+ 'PlatformDetails': 'Linux/UNIX', 'UsageOperation': 'RunInstances',
+ 'State': 'available', 'BlockDeviceMappings': [
+ {'DeviceName': '/dev/sda1',
+ 'Ebs': {'DeleteOnTermination': True,
+ 'SnapshotId': 'snap-0a2df75621efaf18c',
+ 'VolumeSize': 30,
+ 'VolumeType': 'gp2',
+ 'Encrypted': False}},
+ {'DeviceName': '/dev/sdb',
+ 'VirtualName': 'ephemeral0'},
+ {'DeviceName': '/dev/sdc',
+ 'VirtualName': 'ephemeral1'},
+ {'DeviceName': '/dev/sdd',
+ 'VirtualName': 'ephemeral2'},
+ {'DeviceName': '/dev/sde',
+ 'VirtualName': 'ephemeral3'},
+ {'DeviceName': '/dev/sdf',
+ 'VirtualName': 'ephemeral4'},
+ {'DeviceName': '/dev/sdg',
+ 'VirtualName': 'ephemeral5'},
+ {'DeviceName': '/dev/sdh',
+ 'VirtualName': 'ephemeral6'},
+ {'DeviceName': '/dev/sdi',
+ 'VirtualName': 'ephemeral7'}],
+ 'Description': 'ScyllaDB 4.4.1', 'EnaSupport': True, 'Hypervisor': 'xen',
+ 'Name': 'ScyllaDB 4.4.1', 'RootDeviceName': '/dev/sda1', 'RootDeviceType': 'ebs',
+ 'Tags': [
+ {'Key': 'ScyllaMachineImageVersion', 'Value': '4.4.1-20210407.3e5b0f86c2'},
+ {'Key': 'ScyllaPython3Version', 'Value': '4.4.1-0.20210406.00da6b5e9'},
+ {'Key': 'user_data_format_version', 'Value': '2'},
+ {'Key': 'ScyllaToolsVersion', 'Value': '4.4.1-0.20210406.00da6b5e9'},
+ {'Key': 'ScyllaJMXVersion', 'Value': '4.4.1-0.20210406.00da6b5e9'},
+ {'Key': 'branch', 'Value': 'branch-4.4'},
+ {'Key': 'scylla-git-commit', 'Value': '1439d48e2a2b5a3841a87b47f1f52d6fb904a902'},
+ {'Key': 'build-tag', 'Value': 'jenkins-scylla-4.4-ami-52'},
+ {'Key': 'ScyllaVersion', 'Value': '4.4.1-0.20210406.00da6b5e9'},
+ {'Key': 'build-id', 'Value': '52'}
+ ], 'VirtualizationType': 'hvm'})
+
+ @property
+ def temp_dir(self):
+ return tempfile.mkdtemp()
+
+ def _run_test(self, config_path: str, expected_node_config: str):
+ old_environ = os.environ.copy()
+ old_test_config = {key: value for key, value in TestConfig().__class__.__dict__.items() if
+ is_builtin(type(value)) and not key.startswith('__')}
+ while os.environ:
+ os.environ.popitem()
+ try:
+ with patch("sdcm.sct_config.get_scylla_ami_versions", return_value=[self.get_scylla_ami_version_output]), \
+ patch("sdcm.provision.scylla_yaml.certificate_builder.install_client_certificate",
+ return_value=None):
+ os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
+ os.environ['SCT_REGION_NAME'] = '["eu-west-1", "us-east-1"]'
+ os.environ['SCT_CONFIG_FILES'] = config_path
+
+ conf = SCTConfiguration()
+ test_config = TestConfig()
+
+ cluster_backend = conf.get('cluster_backend')
+ if cluster_backend == 'aws':
+ test_config.set_multi_region(len(conf.get('region_name').split()) > 1)
+ elif cluster_backend == 'gce':
+ test_config.set_multi_region(len(conf.get('gce_datacenter').split()) > 1)
+
+ test_config.set_ip_ssh_connections(conf.get(key='ip_ssh_connections'))
+ test_config.set_intra_node_comm_public(conf.get('intra_node_comm_public') or test_config.MULTI_REGION)
+
+ cluster = DummyCluster(params=conf)
+ for node_num in range(3):
+ node = DummyNode(node_num=node_num + 1, parent_cluster=cluster, base_logdir=self.temp_dir)
+ node.init()
+ node.is_seed = False
+ cluster.nodes.append(node)
+ cluster.nodes[0].is_seed = True
+ cluster.init_nodes()
+ seed_node_ips = ','.join(cluster.seed_nodes_ips)
+ for node in cluster.nodes:
+ node.validate_scylla_yaml(expected_node_config=expected_node_config, seed_node_ips=seed_node_ips)
+ finally:
+ while os.environ:
+ os.environ.popitem()
+ os.environ.update(old_environ)
+ for key, value in old_test_config.items():
+ setattr(TestConfig, key, value)
+
+ @parameterized.expand([
+ (
+ os.path.join(BASE_FOLDER, config_name),
+ os.path.join(BASE_FOLDER, config_name.replace('.yaml', '.result.json')),
+ ) for config_name in os.listdir(BASE_FOLDER) if config_name.endswith('.yaml')
+ ])
+ def test_integration_node(self, config_path, result_path):
+ with open(result_path, 'r') as result_file:
+ expected_node_config = result_file.read()
+ self._run_test(config_path, expected_node_config=expected_node_config)

Reply all

Reply to author

Forward

0 new messages