From a253665c5ae9ad4d3a950f4025607fa61ce865f5 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:07:25 +0900 Subject: [PATCH 01/17] =?UTF-8?q?chore:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20secret=20=ED=8F=AC=EC=9D=B8=ED=84=B0=20?= =?UTF-8?q?=EA=B0=B1=EC=8B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: 부하 테스트 실행에 필요한 secret submodule 변경 커밋을 상위 인프라 저장소에 반영 --- config/secrets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/secrets b/config/secrets index f88a84c..b11ecad 160000 --- a/config/secrets +++ b/config/secrets @@ -1 +1 @@ -Subproject commit f88a84cdab72136d294614fd1e2c855c4a026c43 +Subproject commit b11ecadb9329eb4e84b93e98e94d8ab685605caa From d7456eccf62c1d73c0fd7cbf07dbfcaa806b3f7e Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:07:31 +0900 Subject: [PATCH 02/17] =?UTF-8?q?feat:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20RDS=20=EC=9D=B8=ED=94=84=EB=9D=BC=20?= =?UTF-8?q?=EA=B5=AC=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: 부하 테스트용 RDS, 보안 그룹, SSM datasource 파라미터를 Terraform으로 정의 - 상세내용: prod/stage EC2 보안 그룹에서 loadtest RDS 3306 접근을 허용하도록 구성 --- environment/load_test/main.tf | 145 ++++++++++++++++++++++++++++- environment/load_test/output.tf | 49 ++++++++++ environment/load_test/provider.tf | 19 ++++ environment/load_test/variables.tf | 77 ++++++++++++++- 4 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 environment/load_test/output.tf diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index 995074f..72b1c58 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -1 +1,144 @@ -# TODO:: 부하 테스트용 EC2 인스턴스 및 보안 그룹 리소스 정의 필요 +data "aws_vpc" "default" { + default = true +} + +data "aws_subnets" "default" { + filter { + name = "vpc-id" + values = [data.aws_vpc.default.id] + } +} + +data "aws_instance" "prod_api" { + filter { + name = "tag:Name" + values = [var.prod_api_instance_name] + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + +data "aws_instance" "stage_api" { + filter { + name = "tag:Name" + values = [var.stage_api_instance_name] + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + +data "aws_db_instance" "prod" { + db_instance_identifier = var.prod_rds_identifier +} + +data "aws_ssm_parameter" "db_root_username" { + name = var.load_test_db_username_parameter_name +} + +data "aws_ssm_parameter" "db_root_password" { + name = var.load_test_db_password_parameter_name + with_decryption = true +} + +locals { + db_root_username = data.aws_ssm_parameter.db_root_username.value + db_root_password = data.aws_ssm_parameter.db_root_password.value + + source_security_group_ids = setunion( + data.aws_instance.prod_api.vpc_security_group_ids, + data.aws_instance.stage_api.vpc_security_group_ids + ) +} + +resource "aws_security_group" "load_test_db" { + name = "sc-load-test-db-sg" + description = "Security group for load test RDS" + vpc_id = data.aws_vpc.default.id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "solid-connection-load-test-db-sg" + } +} + +resource "aws_security_group_rule" "load_test_db_mysql" { + for_each = local.source_security_group_ids + + type = "ingress" + description = "MySQL from prod/stage API server" + from_port = 3306 + to_port = 3306 + protocol = "tcp" + security_group_id = aws_security_group.load_test_db.id + source_security_group_id = each.value +} + +resource "aws_db_subnet_group" "load_test" { + name = "sc-load-test-db-subnet-group" + subnet_ids = data.aws_subnets.default.ids + + tags = { + Name = "solid-connection-load-test-db-subnet-group" + } +} + +resource "aws_db_instance" "load_test" { + identifier = var.rds_identifier + allocated_storage = var.allocated_storage + engine = "mysql" + engine_version = var.db_engine_version + instance_class = var.db_instance_class + db_name = var.db_name + username = local.db_root_username + password = local.db_root_password + parameter_group_name = var.db_parameter_group_name + db_subnet_group_name = aws_db_subnet_group.load_test.name + vpc_security_group_ids = [aws_security_group.load_test_db.id] + publicly_accessible = false + skip_final_snapshot = true + copy_tags_to_snapshot = true + deletion_protection = false + backup_retention_period = 0 + apply_immediately = true + storage_encrypted = true + kms_key_id = var.kms_key_arn + + tags = { + Name = var.rds_identifier + } +} + +resource "aws_ssm_parameter" "load_test_datasource_url" { + name = "${var.load_test_parameter_prefix}/spring.datasource.url" + type = "String" + value = "jdbc:mysql://${aws_db_instance.load_test.address}:${aws_db_instance.load_test.port}/${var.db_name}?serverTimezone=Asia/Seoul&characterEncoding=UTF-8" + overwrite = true +} + +resource "aws_ssm_parameter" "load_test_datasource_username" { + name = "${var.load_test_parameter_prefix}/spring.datasource.username" + type = "String" + value = local.db_root_username + overwrite = true +} + +resource "aws_ssm_parameter" "load_test_datasource_password" { + name = "${var.load_test_parameter_prefix}/spring.datasource.password" + type = "SecureString" + value = local.db_root_password + key_id = var.ssm_kms_key_id + overwrite = true + tier = "Standard" +} diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf new file mode 100644 index 0000000..55390ac --- /dev/null +++ b/environment/load_test/output.tf @@ -0,0 +1,49 @@ +output "load_test_rds_endpoint" { + description = "Load test RDS endpoint" + value = aws_db_instance.load_test.address +} + +output "load_test_rds_port" { + description = "Load test RDS port" + value = aws_db_instance.load_test.port +} + +output "load_test_rds_identifier" { + description = "Load test RDS identifier" + value = aws_db_instance.load_test.identifier +} + +output "load_test_db_name" { + description = "Load test database name" + value = var.db_name +} + +output "prod_rds_endpoint" { + description = "Prod RDS endpoint used as dump source" + value = data.aws_db_instance.prod.address +} + +output "prod_rds_port" { + description = "Prod RDS port" + value = data.aws_db_instance.prod.port +} + +output "prod_api_instance_id" { + description = "Prod API EC2 instance ID used to run migration commands" + value = data.aws_instance.prod_api.id +} + +output "stage_api_instance_id" { + description = "Stage API EC2 instance ID" + value = data.aws_instance.stage_api.id +} + +output "stage_api_public_ip" { + description = "Stage API EC2 public IP" + value = data.aws_instance.stage_api.public_ip +} + +output "load_test_ssm_parameter_prefix" { + description = "SSM Parameter Store prefix for load test datasource values" + value = var.load_test_parameter_prefix +} diff --git a/environment/load_test/provider.tf b/environment/load_test/provider.tf index 3c3f8d1..8b41b3d 100644 --- a/environment/load_test/provider.tf +++ b/environment/load_test/provider.tf @@ -1,3 +1,22 @@ +terraform { + required_version = ">= 1.10.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "solid-connection-tfstate" + key = "env/load_test/terraform.tfstate" + region = "ap-northeast-2" + use_lockfile = true + encrypt = true + } +} + provider "aws" { region = "ap-northeast-2" default_tags { diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index 6f74e1f..7d4d639 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -1 +1,76 @@ -# TODO:: 부하 테스트 인스턴스용 변수 정의 +variable "rds_identifier" { + description = "RDS identifier for load test" + type = string +} + +variable "db_instance_class" { + description = "RDS instance class for load test" + type = string +} + +variable "allocated_storage" { + description = "RDS storage in GiB" + type = number + default = 20 +} + +variable "db_engine_version" { + description = "MySQL engine version" + type = string +} + +variable "db_parameter_group_name" { + description = "MySQL parameter group name" + type = string +} + +variable "db_name" { + description = "Application database name" + type = string + default = "solid_connection" +} + +variable "load_test_db_username_parameter_name" { + description = "SSM parameter name containing the load test DB root username" + type = string +} + +variable "load_test_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the load test DB root password" + type = string +} + +variable "kms_key_arn" { + description = "KMS key ARN for RDS storage encryption" + type = string +} + +variable "ssm_kms_key_id" { + description = "KMS key ID or ARN for SSM SecureString. Null uses the AWS managed aws/ssm key." + type = string + default = null + nullable = true +} + +variable "prod_rds_identifier" { + description = "Source prod RDS identifier" + type = string +} + +variable "prod_api_instance_name" { + description = "Name tag of the prod API EC2 instance used to run dump/restore" + type = string + default = "solid-connection-server-prod" +} + +variable "stage_api_instance_name" { + description = "Name tag of the stage API EC2 instance that will connect to load test RDS" + type = string + default = "solid-connection-server-stage" +} + +variable "load_test_parameter_prefix" { + description = "SSM Parameter Store prefix for load test datasource values" + type = string + default = "/solid-connection/loadtest" +} From 95da845031ce00a915f26dfb6bea3d7a80d9c21d Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:07:36 +0900 Subject: [PATCH 03/17] =?UTF-8?q?feat:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=8B=A4=ED=96=89=20=EC=8A=A4=ED=81=AC?= =?UTF-8?q?=EB=A6=BD=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: start.sh에서 RDS 생성, stage 전환, prod 데이터 복사를 자동화 - 상세내용: stop.sh에서 stage 원복과 loadtest RDS destroy 흐름을 제공 - 상세내용: Windows와 macOS/Linux 실행 환경에서 사용할 bash 기반 절차를 문서화 --- scripts/load_test/README.md | 49 +++++++ scripts/load_test/start.sh | 264 ++++++++++++++++++++++++++++++++++++ scripts/load_test/stop.sh | 140 +++++++++++++++++++ 3 files changed, 453 insertions(+) create mode 100644 scripts/load_test/README.md create mode 100644 scripts/load_test/start.sh create mode 100644 scripts/load_test/stop.sh diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md new file mode 100644 index 0000000..2e4d129 --- /dev/null +++ b/scripts/load_test/README.md @@ -0,0 +1,49 @@ +# Load Test Automation + +This automation creates a temporary load test RDS instance, copies prod RDS data +into it, writes load test datasource values to Parameter Store, and optionally +stops/starts the stage application through SSM Run Command. + +## Flow + +1. `Start-LoadTest.ps1` runs `terraform apply` in `environment/load_test`. +2. Terraform creates the load test RDS and writes: + - `/solid-connection/loadtest/spring.datasource.url` + - `/solid-connection/loadtest/spring.datasource.username` + - `/solid-connection/loadtest/spring.datasource.password` +3. The script stores DB migration credentials in temporary SSM parameters. +4. The prod EC2 instance runs `mysqldump` against prod RDS and restores it into + the load test RDS. +5. The optional stage stop command can pause the stage app before the load test. +6. `Stop-LoadTest.ps1` can run an optional stage start command and then destroy + only the load test Terraform stack. + +## Example + +```bash +scripts/load_test/start.sh \ + --switch-stage-to-loadtest \ + --stage-ssh-key ./stage-key.pem +``` + +```bash +scripts/load_test/stop.sh \ + --restore-stage-dev \ + --stage-ssh-key ./stage-key.pem +``` + +## Notes + +- The prod and stage EC2 instances are looked up by their `Name` tags. +- Prod DB username/password are read from Parameter Store. The default paths are + `/solid-connection/prod/spring.datasource.username` and + `/solid-connection/prod/spring.datasource.password`. +- Load test DB username/password are also read from Parameter Store. The default + paths are `/solid-connection/loadtest/spring.datasource.username` and + `/solid-connection/loadtest/spring.datasource.password`. +- The load test RDS security group allows MySQL only from the security groups + attached to the prod and stage API EC2 instances. +- The prod EC2 instance must have SSM access and permission to read the temporary + migration parameters. +- Keep the real `load_test.tfvars` in the secret submodule or another ignored + local location. Do not commit it. diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh new file mode 100644 index 0000000..d962215 --- /dev/null +++ b/scripts/load_test/start.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash +set -euo pipefail + +TERRAFORM_DIR="environment/load_test" +VAR_FILE="../../config/secrets/load_test.tfvars" +DATABASE_NAME="solid_connection" +MIGRATION_PARAMETER_PREFIX="/solid-connection/loadtest/migration" +PROD_DB_USERNAME_PARAMETER="/solid-connection/prod/spring.datasource.username" +PROD_DB_PASSWORD_PARAMETER="/solid-connection/prod/spring.datasource.password" +LOADTEST_DB_USERNAME_PARAMETER="/solid-connection/loadtest/spring.datasource.username" +LOADTEST_DB_PASSWORD_PARAMETER="/solid-connection/loadtest/spring.datasource.password" +SWITCH_STAGE_TO_LOADTEST="false" +STAGE_SSH_USER="ubuntu" +STAGE_SSH_KEY="" +STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" +STAGE_COMPOSE_FILE="docker-compose.dev.yml" +SKIP_TERRAFORM_APPLY="false" +SKIP_DATA_COPY="false" + +usage() { + cat <<'EOF' +Usage: scripts/load_test/start.sh [options] + +Options: + --terraform-dir PATH Default: environment/load_test + --var-file PATH Default: ../../config/secrets/load_test.tfvars + --prod-db-username-parameter Default: /solid-connection/prod/spring.datasource.username + --prod-db-password-parameter Default: /solid-connection/prod/spring.datasource.password + --loadtest-db-username-parameter Default: /solid-connection/loadtest/spring.datasource.username + --loadtest-db-password-parameter Default: /solid-connection/loadtest/spring.datasource.password + --database-name VALUE Default: solid_connection + --migration-prefix VALUE Default: /solid-connection/loadtest/migration + --switch-stage-to-loadtest Restart stage app over SSH with dev,loadtest profiles + --stage-ssh-user VALUE Default: ubuntu + --stage-ssh-key PATH Required with --switch-stage-to-loadtest + --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev + --stage-compose-file VALUE Default: docker-compose.dev.yml + --skip-terraform-apply + --skip-data-copy + -h, --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --var-file) VAR_FILE="$2"; shift 2 ;; + --prod-db-username-parameter) PROD_DB_USERNAME_PARAMETER="$2"; shift 2 ;; + --prod-db-password-parameter) PROD_DB_PASSWORD_PARAMETER="$2"; shift 2 ;; + --loadtest-db-username-parameter) LOADTEST_DB_USERNAME_PARAMETER="$2"; shift 2 ;; + --loadtest-db-password-parameter) LOADTEST_DB_PASSWORD_PARAMETER="$2"; shift 2 ;; + --database-name) DATABASE_NAME="$2"; shift 2 ;; + --migration-prefix) MIGRATION_PARAMETER_PREFIX="$2"; shift 2 ;; + --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; + --stage-ssh-user) STAGE_SSH_USER="$2"; shift 2 ;; + --stage-ssh-key) STAGE_SSH_KEY="$2"; shift 2 ;; + --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; + --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; + --skip-data-copy) SKIP_DATA_COPY="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +require_value() { + local name="$1" + local value="$2" + if [[ -z "$value" ]]; then + echo "Missing required option: $name" >&2 + exit 1 + fi +} + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Required command not found: $1" >&2 + exit 1 + fi +} + +require_command terraform +require_command aws +require_command jq +require_command ssh + +tf_output() { + terraform -chdir="$TERRAFORM_DIR" output -raw "$1" +} + +send_ssm_command() { + local instance_id="$1" + local comment="$2" + local commands_json="$3" + + local command_id + command_id="$(aws ssm send-command \ + --instance-ids "$instance_id" \ + --document-name "AWS-RunShellScript" \ + --comment "$comment" \ + --parameters "$commands_json" \ + --query "Command.CommandId" \ + --output text)" + + local status + while true; do + sleep 5 + status="$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "Status" \ + --output text 2>/dev/null || true)" + + case "$status" in + Pending|InProgress|Delayed|"") continue ;; + Success) break ;; + *) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command failed with status $status: $comment" >&2 + exit 1 + ;; + esac + done +} + +delete_temp_parameters() { + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" >/dev/null 2>&1 || true + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" >/dev/null 2>&1 || true + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-username" >/dev/null 2>&1 || true + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-password" >/dev/null 2>&1 || true +} + +if [[ "$SKIP_TERRAFORM_APPLY" != "true" ]]; then + terraform -chdir="$TERRAFORM_DIR" init + terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" +fi + +prod_instance_id="$(tf_output prod_api_instance_id)" +stage_instance_id="$(tf_output stage_api_instance_id)" +stage_public_ip="$(tf_output stage_api_public_ip)" +prod_endpoint="$(tf_output prod_rds_endpoint)" +prod_port="$(tf_output prod_rds_port)" +loadtest_endpoint="$(tf_output load_test_rds_endpoint)" +loadtest_port="$(tf_output load_test_rds_port)" + +if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then + require_value "--stage-ssh-key" "$STAGE_SSH_KEY" + + ssh -i "$STAGE_SSH_KEY" \ + -o StrictHostKeyChecking=no \ + "$STAGE_SSH_USER@$stage_public_ip" \ + "APP_DIR='$STAGE_APP_DIR' COMPOSE_FILE='$STAGE_COMPOSE_FILE' bash -s" <<'REMOTE' +set -euo pipefail + +cd "$APP_DIR" + +CURRENT_IMAGE="$(docker inspect -f '{{.Config.Image}}' solid-connection-dev 2>/dev/null || true)" +if [[ -z "$CURRENT_IMAGE" ]]; then + echo "solid-connection-dev container is not running; cannot infer image tag" >&2 + exit 1 +fi + +OWNER_LOWERCASE="$(echo "$CURRENT_IMAGE" | sed -E 's#^ghcr.io/([^/]+)/.*#\1#')" +IMAGE_TAG="$(echo "$CURRENT_IMAGE" | sed -E 's#.*:([^:]+)$#\1#')" + +cat > docker-compose.loadtest.override.yml <<'YAML' +services: + solid-connection-dev: + environment: + - SPRING_PROFILES_ACTIVE=dev,loadtest + - AWS_REGION=ap-northeast-2 + - SPRING_DATA_REDIS_HOST=127.0.0.1 + - SPRING_DATA_REDIS_PORT=6379 +YAML + +docker compose -f "$COMPOSE_FILE" -f docker-compose.loadtest.override.yml down || true +OWNER_LOWERCASE="$OWNER_LOWERCASE" IMAGE_TAG="$IMAGE_TAG" \ + docker compose -f "$COMPOSE_FILE" -f docker-compose.loadtest.override.yml up -d solid-connection-dev +REMOTE +fi + +if [[ "$SKIP_DATA_COPY" != "true" ]]; then + trap delete_temp_parameters EXIT + + prod_db_username="$(aws ssm get-parameter \ + --name "$PROD_DB_USERNAME_PARAMETER" \ + --query "Parameter.Value" \ + --output text)" + + prod_db_password="$(aws ssm get-parameter \ + --name "$PROD_DB_PASSWORD_PARAMETER" \ + --with-decryption \ + --query "Parameter.Value" \ + --output text)" + + loadtest_db_username="$(aws ssm get-parameter \ + --name "$LOADTEST_DB_USERNAME_PARAMETER" \ + --query "Parameter.Value" \ + --output text)" + + loadtest_db_password="$(aws ssm get-parameter \ + --name "$LOADTEST_DB_PASSWORD_PARAMETER" \ + --with-decryption \ + --query "Parameter.Value" \ + --output text)" + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" \ + --type String \ + --value "$prod_db_username" \ + --overwrite >/dev/null + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" \ + --type SecureString \ + --value "$prod_db_password" \ + --overwrite >/dev/null + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-username" \ + --type String \ + --value "$loadtest_db_username" \ + --overwrite >/dev/null + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-password" \ + --type SecureString \ + --value "$loadtest_db_password" \ + --overwrite >/dev/null + + copy_commands_json="$(jq -cn \ + --arg prefix "$MIGRATION_PARAMETER_PREFIX" \ + --arg prod_endpoint "$prod_endpoint" \ + --arg prod_port "$prod_port" \ + --arg loadtest_endpoint "$loadtest_endpoint" \ + --arg loadtest_port "$loadtest_port" \ + --arg database "$DATABASE_NAME" \ + '{ + commands: [ + "set -euo pipefail", + "export DEBIAN_FRONTEND=noninteractive", + "if ! command -v mysqldump >/dev/null 2>&1 || ! command -v mysql >/dev/null 2>&1; then sudo apt-get update && sudo apt-get install -y mysql-client; fi", + "PROD_USER=$(aws ssm get-parameter --name \($prefix)/prod-db-username --query Parameter.Value --output text)", + "PROD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/prod-db-password --with-decryption --query Parameter.Value --output text)", + "LOAD_USER=$(aws ssm get-parameter --name \($prefix)/loadtest-db-username --query Parameter.Value --output text)", + "LOAD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/loadtest-db-password --with-decryption --query Parameter.Value --output text)", + "DUMP_FILE=/tmp/solid-connection-loadtest-$(date +%Y%m%d%H%M%S).sql.gz", + "MYSQL_PWD=\"$PROD_PASSWORD\" mysqldump --single-transaction --set-gtid-purged=OFF --column-statistics=0 -h \($prod_endpoint) -P \($prod_port) -u \"$PROD_USER\" \($database) | gzip > \"$DUMP_FILE\"", + "MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" -e \"DROP DATABASE IF EXISTS \\\`\($database)\\\`; CREATE DATABASE \\\`\($database)\\\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;\"", + "gunzip -c \"$DUMP_FILE\" | MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" \($database)", + "rm -f \"$DUMP_FILE\"" + ] + }')" + + send_ssm_command "$prod_instance_id" "Copy prod RDS data to load test RDS" "$copy_commands_json" +fi + +echo "Load test environment is ready." +echo "RDS endpoint: ${loadtest_endpoint}:${loadtest_port}" +echo "Stage instance: ${stage_instance_id}" +echo "Stage public IP: ${stage_public_ip}" diff --git a/scripts/load_test/stop.sh b/scripts/load_test/stop.sh new file mode 100644 index 0000000..f0857ec --- /dev/null +++ b/scripts/load_test/stop.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +set -euo pipefail + +TERRAFORM_DIR="environment/load_test" +VAR_FILE="../../config/secrets/load_test.tfvars" +STAGE_START_COMMAND="" +RESTORE_STAGE_DEV="false" +STAGE_SSH_USER="ubuntu" +STAGE_SSH_KEY="" +STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" +STAGE_COMPOSE_FILE="docker-compose.dev.yml" +SKIP_TERRAFORM_DESTROY="false" + +usage() { + cat <<'EOF' +Usage: scripts/load_test/stop.sh [options] + +Options: + --terraform-dir PATH Default: environment/load_test + --var-file PATH Default: ../../config/secrets/load_test.tfvars + --restore-stage-dev Restart stage app over SSH with dev profile + --stage-ssh-user VALUE Default: ubuntu + --stage-ssh-key PATH Required with --restore-stage-dev + --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev + --stage-compose-file VALUE Default: docker-compose.dev.yml + --skip-terraform-destroy + -h, --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --var-file) VAR_FILE="$2"; shift 2 ;; + --restore-stage-dev) RESTORE_STAGE_DEV="true"; shift ;; + --stage-ssh-user) STAGE_SSH_USER="$2"; shift 2 ;; + --stage-ssh-key) STAGE_SSH_KEY="$2"; shift 2 ;; + --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; + --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --skip-terraform-destroy) SKIP_TERRAFORM_DESTROY="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Required command not found: $1" >&2 + exit 1 + fi +} + +require_command terraform +require_command aws +require_command jq +require_command ssh + +require_value() { + local name="$1" + local value="$2" + if [[ -z "$value" ]]; then + echo "Missing required option: $name" >&2 + exit 1 + fi +} + +tf_output() { + terraform -chdir="$TERRAFORM_DIR" output -raw "$1" +} + +send_ssm_command() { + local instance_id="$1" + local comment="$2" + local commands_json="$3" + + local command_id + command_id="$(aws ssm send-command \ + --instance-ids "$instance_id" \ + --document-name "AWS-RunShellScript" \ + --comment "$comment" \ + --parameters "$commands_json" \ + --query "Command.CommandId" \ + --output text)" + + local status + while true; do + sleep 5 + status="$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "Status" \ + --output text 2>/dev/null || true)" + + case "$status" in + Pending|InProgress|Delayed|"") continue ;; + Success) break ;; + *) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command failed with status $status: $comment" >&2 + exit 1 + ;; + esac + done +} + +if [[ "$RESTORE_STAGE_DEV" == "true" ]]; then + require_value "--stage-ssh-key" "$STAGE_SSH_KEY" + stage_public_ip="$(tf_output stage_api_public_ip)" + + ssh -i "$STAGE_SSH_KEY" \ + -o StrictHostKeyChecking=no \ + "$STAGE_SSH_USER@$stage_public_ip" \ + "APP_DIR='$STAGE_APP_DIR' COMPOSE_FILE='$STAGE_COMPOSE_FILE' bash -s" <<'REMOTE' +set -euo pipefail + +cd "$APP_DIR" + +CURRENT_IMAGE="$(docker inspect -f '{{.Config.Image}}' solid-connection-dev 2>/dev/null || true)" +if [[ -z "$CURRENT_IMAGE" ]]; then + echo "solid-connection-dev container is not running; cannot infer image tag" >&2 + exit 1 +fi + +OWNER_LOWERCASE="$(echo "$CURRENT_IMAGE" | sed -E 's#^ghcr.io/([^/]+)/.*#\1#')" +IMAGE_TAG="$(echo "$CURRENT_IMAGE" | sed -E 's#.*:([^:]+)$#\1#')" + +rm -f docker-compose.loadtest.override.yml +docker compose -f "$COMPOSE_FILE" down || true +OWNER_LOWERCASE="$OWNER_LOWERCASE" IMAGE_TAG="$IMAGE_TAG" docker compose -f "$COMPOSE_FILE" up -d +REMOTE +fi + +if [[ "$SKIP_TERRAFORM_DESTROY" != "true" ]]; then + terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" +fi + +echo "Load test environment has been stopped." From bdfb439882efaf23319beb346213199019404a3a Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:47:00 +0900 Subject: [PATCH 04/17] =?UTF-8?q?feat:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20GitHub=20Actions=20=EC=8B=A4=ED=96=89=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: workflow_dispatch로 부하 테스트 시작과 종료를 수동 실행할 수 있도록 워크플로우 추가 - 상세내용: stage 서버 전환과 원복을 SSH 대신 SSM RunCommand로 수행하도록 변경 - 상세내용: SSH key 입력 없이 OIDC 기반 AWS Role과 GH_PAT submodule checkout 흐름을 사용하도록 문서화 --- .github/workflows/load-test-start.yml | 65 +++++++++++++++++++++++ .github/workflows/load-test-stop.yml | 65 +++++++++++++++++++++++ scripts/load_test/README.md | 74 ++++++++++++++++----------- scripts/load_test/start.sh | 58 +++++++-------------- scripts/load_test/stop.sh | 65 +++++++++-------------- 5 files changed, 214 insertions(+), 113 deletions(-) create mode 100644 .github/workflows/load-test-start.yml create mode 100644 .github/workflows/load-test-stop.yml diff --git a/.github/workflows/load-test-start.yml b/.github/workflows/load-test-start.yml new file mode 100644 index 0000000..80125e4 --- /dev/null +++ b/.github/workflows/load-test-start.yml @@ -0,0 +1,65 @@ +name: Load Test Start + +on: + workflow_dispatch: + inputs: + switch_stage_to_loadtest: + description: "Restart stage app with dev,loadtest profiles" + required: true + default: true + type: boolean + copy_prod_data: + description: "Copy prod RDS data to load test RDS" + required: true + default: true + type: boolean + +permissions: + id-token: write + contents: read + +concurrency: + group: load-test-environment + cancel-in-progress: false + +env: + TF_VERSION: "1.10.5" + +jobs: + start: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ap-northeast-2 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Start load test environment + run: | + args=() + + if [ "${{ inputs.switch_stage_to_loadtest }}" = "true" ]; then + args+=(--switch-stage-to-loadtest) + fi + + if [ "${{ inputs.copy_prod_data }}" != "true" ]; then + args+=(--skip-data-copy) + fi + + bash scripts/load_test/start.sh "${args[@]}" diff --git a/.github/workflows/load-test-stop.yml b/.github/workflows/load-test-stop.yml new file mode 100644 index 0000000..9e5fba7 --- /dev/null +++ b/.github/workflows/load-test-stop.yml @@ -0,0 +1,65 @@ +name: Load Test Stop + +on: + workflow_dispatch: + inputs: + restore_stage_dev: + description: "Restart stage app with dev profile" + required: true + default: true + type: boolean + destroy_rds: + description: "Destroy load test Terraform stack" + required: true + default: true + type: boolean + +permissions: + id-token: write + contents: read + +concurrency: + group: load-test-environment + cancel-in-progress: false + +env: + TF_VERSION: "1.10.5" + +jobs: + stop: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ap-northeast-2 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Stop load test environment + run: | + args=() + + if [ "${{ inputs.restore_stage_dev }}" = "true" ]; then + args+=(--restore-stage-dev) + fi + + if [ "${{ inputs.destroy_rds }}" != "true" ]; then + args+=(--skip-terraform-destroy) + fi + + bash scripts/load_test/stop.sh "${args[@]}" diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 2e4d129..0283669 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -1,49 +1,61 @@ # Load Test Automation This automation creates a temporary load test RDS instance, copies prod RDS data -into it, writes load test datasource values to Parameter Store, and optionally -stops/starts the stage application through SSM Run Command. +into it, writes load test datasource values to Parameter Store, and switches the +stage application through SSM Run Command. -## Flow +## GitHub Actions Flow -1. `Start-LoadTest.ps1` runs `terraform apply` in `environment/load_test`. -2. Terraform creates the load test RDS and writes: +### Start + +1. Open **Actions > Load Test Start**. +2. Click **Run workflow**. +3. Keep `switch_stage_to_loadtest` enabled to restart stage with + `dev,loadtest` profiles. +4. Keep `copy_prod_data` enabled to copy prod RDS data into the load test RDS. + +The workflow runs `scripts/load_test/start.sh`. + +### Stop + +1. Open **Actions > Load Test Stop**. +2. Click **Run workflow**. +3. Keep `restore_stage_dev` enabled to restart stage with the normal dev + compose configuration. +4. Keep `destroy_rds` enabled to destroy the load test Terraform stack. + +The workflow runs `scripts/load_test/stop.sh`. + +## What Start Does + +1. Runs `terraform apply` in `environment/load_test`. +2. Creates the load test RDS and writes: - `/solid-connection/loadtest/spring.datasource.url` - `/solid-connection/loadtest/spring.datasource.username` - `/solid-connection/loadtest/spring.datasource.password` -3. The script stores DB migration credentials in temporary SSM parameters. -4. The prod EC2 instance runs `mysqldump` against prod RDS and restores it into - the load test RDS. -5. The optional stage stop command can pause the stage app before the load test. -6. `Stop-LoadTest.ps1` can run an optional stage start command and then destroy - only the load test Terraform stack. - -## Example - -```bash -scripts/load_test/start.sh \ - --switch-stage-to-loadtest \ - --stage-ssh-key ./stage-key.pem -``` - -```bash -scripts/load_test/stop.sh \ - --restore-stage-dev \ - --stage-ssh-key ./stage-key.pem -``` +3. Switches the stage app to `dev,loadtest` profiles through SSM Run Command. +4. Stores DB migration credentials in temporary SSM parameters. +5. Runs `mysqldump` on the prod EC2 instance through SSM Run Command and + restores the dump into the load test RDS. +6. Deletes the temporary migration parameters. + +## What Stop Does + +1. Restores the stage app to the normal dev compose configuration through SSM + Run Command. +2. Runs `terraform destroy` for the load test Terraform stack. ## Notes +- GitHub Actions uses `AWS_ROLE_ARN` through OIDC. +- Private submodule checkout uses `GH_PAT`. +- No SSH private key is required for load test start/stop. - The prod and stage EC2 instances are looked up by their `Name` tags. - Prod DB username/password are read from Parameter Store. The default paths are `/solid-connection/prod/spring.datasource.username` and `/solid-connection/prod/spring.datasource.password`. -- Load test DB username/password are also read from Parameter Store. The default - paths are `/solid-connection/loadtest/spring.datasource.username` and +- Load test DB username/password are read from Parameter Store. The default paths + are `/solid-connection/loadtest/spring.datasource.username` and `/solid-connection/loadtest/spring.datasource.password`. - The load test RDS security group allows MySQL only from the security groups attached to the prod and stage API EC2 instances. -- The prod EC2 instance must have SSM access and permission to read the temporary - migration parameters. -- Keep the real `load_test.tfvars` in the secret submodule or another ignored - local location. Do not commit it. diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh index d962215..1f9c75a 100644 --- a/scripts/load_test/start.sh +++ b/scripts/load_test/start.sh @@ -10,8 +10,6 @@ PROD_DB_PASSWORD_PARAMETER="/solid-connection/prod/spring.datasource.password" LOADTEST_DB_USERNAME_PARAMETER="/solid-connection/loadtest/spring.datasource.username" LOADTEST_DB_PASSWORD_PARAMETER="/solid-connection/loadtest/spring.datasource.password" SWITCH_STAGE_TO_LOADTEST="false" -STAGE_SSH_USER="ubuntu" -STAGE_SSH_KEY="" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" SKIP_TERRAFORM_APPLY="false" @@ -30,9 +28,7 @@ Options: --loadtest-db-password-parameter Default: /solid-connection/loadtest/spring.datasource.password --database-name VALUE Default: solid_connection --migration-prefix VALUE Default: /solid-connection/loadtest/migration - --switch-stage-to-loadtest Restart stage app over SSH with dev,loadtest profiles - --stage-ssh-user VALUE Default: ubuntu - --stage-ssh-key PATH Required with --switch-stage-to-loadtest + --switch-stage-to-loadtest Restart stage app through SSM with dev,loadtest profiles --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml --skip-terraform-apply @@ -52,8 +48,6 @@ while [[ $# -gt 0 ]]; do --database-name) DATABASE_NAME="$2"; shift 2 ;; --migration-prefix) MIGRATION_PARAMETER_PREFIX="$2"; shift 2 ;; --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; - --stage-ssh-user) STAGE_SSH_USER="$2"; shift 2 ;; - --stage-ssh-key) STAGE_SSH_KEY="$2"; shift 2 ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; @@ -82,7 +76,6 @@ require_command() { require_command terraform require_command aws require_command jq -require_command ssh tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" @@ -147,39 +140,24 @@ loadtest_endpoint="$(tf_output load_test_rds_endpoint)" loadtest_port="$(tf_output load_test_rds_port)" if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then - require_value "--stage-ssh-key" "$STAGE_SSH_KEY" - - ssh -i "$STAGE_SSH_KEY" \ - -o StrictHostKeyChecking=no \ - "$STAGE_SSH_USER@$stage_public_ip" \ - "APP_DIR='$STAGE_APP_DIR' COMPOSE_FILE='$STAGE_COMPOSE_FILE' bash -s" <<'REMOTE' -set -euo pipefail - -cd "$APP_DIR" - -CURRENT_IMAGE="$(docker inspect -f '{{.Config.Image}}' solid-connection-dev 2>/dev/null || true)" -if [[ -z "$CURRENT_IMAGE" ]]; then - echo "solid-connection-dev container is not running; cannot infer image tag" >&2 - exit 1 -fi - -OWNER_LOWERCASE="$(echo "$CURRENT_IMAGE" | sed -E 's#^ghcr.io/([^/]+)/.*#\1#')" -IMAGE_TAG="$(echo "$CURRENT_IMAGE" | sed -E 's#.*:([^:]+)$#\1#')" - -cat > docker-compose.loadtest.override.yml <<'YAML' -services: - solid-connection-dev: - environment: - - SPRING_PROFILES_ACTIVE=dev,loadtest - - AWS_REGION=ap-northeast-2 - - SPRING_DATA_REDIS_HOST=127.0.0.1 - - SPRING_DATA_REDIS_PORT=6379 -YAML + stage_commands_json="$(jq -cn \ + --arg app_dir "$STAGE_APP_DIR" \ + --arg compose_file "$STAGE_COMPOSE_FILE" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($app_dir)", + "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", + "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", + "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", + "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", + "cat > docker-compose.loadtest.override.yml <<'\''YAML'\''\nservices:\n solid-connection-dev:\n environment:\n - SPRING_PROFILES_ACTIVE=dev,loadtest\n - AWS_REGION=ap-northeast-2\n - SPRING_DATA_REDIS_HOST=127.0.0.1\n - SPRING_DATA_REDIS_PORT=6379\nYAML", + "docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml down || true", + "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml up -d solid-connection-dev" + ] + }')" -docker compose -f "$COMPOSE_FILE" -f docker-compose.loadtest.override.yml down || true -OWNER_LOWERCASE="$OWNER_LOWERCASE" IMAGE_TAG="$IMAGE_TAG" \ - docker compose -f "$COMPOSE_FILE" -f docker-compose.loadtest.override.yml up -d solid-connection-dev -REMOTE + send_ssm_command "$stage_instance_id" "Switch stage app to load test datasource" "$stage_commands_json" fi if [[ "$SKIP_DATA_COPY" != "true" ]]; then diff --git a/scripts/load_test/stop.sh b/scripts/load_test/stop.sh index f0857ec..62687fb 100644 --- a/scripts/load_test/stop.sh +++ b/scripts/load_test/stop.sh @@ -3,10 +3,7 @@ set -euo pipefail TERRAFORM_DIR="environment/load_test" VAR_FILE="../../config/secrets/load_test.tfvars" -STAGE_START_COMMAND="" RESTORE_STAGE_DEV="false" -STAGE_SSH_USER="ubuntu" -STAGE_SSH_KEY="" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" SKIP_TERRAFORM_DESTROY="false" @@ -18,9 +15,7 @@ Usage: scripts/load_test/stop.sh [options] Options: --terraform-dir PATH Default: environment/load_test --var-file PATH Default: ../../config/secrets/load_test.tfvars - --restore-stage-dev Restart stage app over SSH with dev profile - --stage-ssh-user VALUE Default: ubuntu - --stage-ssh-key PATH Required with --restore-stage-dev + --restore-stage-dev Restart stage app through SSM with dev profile --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml --skip-terraform-destroy @@ -33,8 +28,6 @@ while [[ $# -gt 0 ]]; do --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; --var-file) VAR_FILE="$2"; shift 2 ;; --restore-stage-dev) RESTORE_STAGE_DEV="true"; shift ;; - --stage-ssh-user) STAGE_SSH_USER="$2"; shift 2 ;; - --stage-ssh-key) STAGE_SSH_KEY="$2"; shift 2 ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; --skip-terraform-destroy) SKIP_TERRAFORM_DESTROY="true"; shift ;; @@ -53,16 +46,6 @@ require_command() { require_command terraform require_command aws require_command jq -require_command ssh - -require_value() { - local name="$1" - local value="$2" - if [[ -z "$value" ]]; then - echo "Missing required option: $name" >&2 - exit 1 - fi -} tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" @@ -106,31 +89,29 @@ send_ssm_command() { done } -if [[ "$RESTORE_STAGE_DEV" == "true" ]]; then - require_value "--stage-ssh-key" "$STAGE_SSH_KEY" - stage_public_ip="$(tf_output stage_api_public_ip)" - - ssh -i "$STAGE_SSH_KEY" \ - -o StrictHostKeyChecking=no \ - "$STAGE_SSH_USER@$stage_public_ip" \ - "APP_DIR='$STAGE_APP_DIR' COMPOSE_FILE='$STAGE_COMPOSE_FILE' bash -s" <<'REMOTE' -set -euo pipefail - -cd "$APP_DIR" - -CURRENT_IMAGE="$(docker inspect -f '{{.Config.Image}}' solid-connection-dev 2>/dev/null || true)" -if [[ -z "$CURRENT_IMAGE" ]]; then - echo "solid-connection-dev container is not running; cannot infer image tag" >&2 - exit 1 -fi - -OWNER_LOWERCASE="$(echo "$CURRENT_IMAGE" | sed -E 's#^ghcr.io/([^/]+)/.*#\1#')" -IMAGE_TAG="$(echo "$CURRENT_IMAGE" | sed -E 's#.*:([^:]+)$#\1#')" +terraform -chdir="$TERRAFORM_DIR" init -rm -f docker-compose.loadtest.override.yml -docker compose -f "$COMPOSE_FILE" down || true -OWNER_LOWERCASE="$OWNER_LOWERCASE" IMAGE_TAG="$IMAGE_TAG" docker compose -f "$COMPOSE_FILE" up -d -REMOTE +if [[ "$RESTORE_STAGE_DEV" == "true" ]]; then + stage_instance_id="$(tf_output stage_api_instance_id)" + + stage_commands_json="$(jq -cn \ + --arg app_dir "$STAGE_APP_DIR" \ + --arg compose_file "$STAGE_COMPOSE_FILE" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($app_dir)", + "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", + "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", + "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", + "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", + "rm -f docker-compose.loadtest.override.yml", + "docker compose -f \($compose_file) down || true", + "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) up -d" + ] + }')" + + send_ssm_command "$stage_instance_id" "Restore stage app to dev datasource" "$stage_commands_json" fi if [[ "$SKIP_TERRAFORM_DESTROY" != "true" ]]; then From 9ef254a409cb20073077f84cbf7e23a1618b12d7 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:54:49 +0900 Subject: [PATCH 05/17] =?UTF-8?q?feat:=20stage=20=EC=83=9D=EC=84=B1=20?= =?UTF-8?q?=EC=8B=9C=20k6=20=ED=8C=8C=EC=9D=BC=20=EB=B0=B0=EC=B9=98=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: monitor repo의 k6 파일을 infra repo에 포함해 stage EC2 cloud-init에서 배치하도록 구성 - 상세내용: app_stack module에 k6 파일 배치 옵션을 추가하고 stage 환경에서만 활성화 - 상세내용: 부하 테스트 README를 한글로 변경하고 GitHub Actions 실행 흐름을 정리 --- config/load-test/k6/createPost.json | 7 + config/load-test/k6/script/set-load-test.sh | 64 ++++ config/load-test/k6/set_up_xk6.sh | 53 +++ config/load-test/k6/updatePost.json | 5 + config/load-test/k6/whole-user-flow.js | 372 ++++++++++++++++++++ environment/stage/main.tf | 12 +- modules/app_stack/ec2.tf | 63 +++- modules/app_stack/variables.tf | 14 +- scripts/load_test/README.md | 109 +++--- 9 files changed, 639 insertions(+), 60 deletions(-) create mode 100644 config/load-test/k6/createPost.json create mode 100644 config/load-test/k6/script/set-load-test.sh create mode 100644 config/load-test/k6/set_up_xk6.sh create mode 100644 config/load-test/k6/updatePost.json create mode 100644 config/load-test/k6/whole-user-flow.js diff --git a/config/load-test/k6/createPost.json b/config/load-test/k6/createPost.json new file mode 100644 index 0000000..e08b0d2 --- /dev/null +++ b/config/load-test/k6/createPost.json @@ -0,0 +1,7 @@ +{ + "boardCode": "FREE", + "postCategory": "자유", + "title": "수강신청 어떻게 하나요?", + "content": "수강신청 방법을 잘 모르겠어요.", + "isQuestion": false +} diff --git a/config/load-test/k6/script/set-load-test.sh b/config/load-test/k6/script/set-load-test.sh new file mode 100644 index 0000000..b401b2d --- /dev/null +++ b/config/load-test/k6/script/set-load-test.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +# 작업 디렉터리 설정 +WORKDIR="/home/ubuntu" + +####################################################################### +# set-load-test.sh +# 사용 예: SQL_FILE_BASENAME=regular.sql ./set-load-test.sh +####################################################################### + +# 0. 필수 값 검증 +SQL_BASENAME="${SQL_FILE_BASENAME:-regular.sql}" +if [[ -z "$SQL_BASENAME" ]]; then + echo "❌ 사용할 덤프 파일이 입력되지 않았습니다." >&2 + echo "SQL_FILE_BASENAME={dump 파일 이름} ./set-load-test.sh 형식으로 인자를 전달해야합니다." >&2 + exit 1 +fi + +SQL_SRC="$WORKDIR/load-test-setting/db/${SQL_BASENAME}" +if [[ ! -f "$SQL_SRC" ]]; then + echo "❌ 덤프 파일이 파일 시스템에 존재하지 않습니다: $SQL_SRC" >&2 + exit 2 +fi + +# 1. 기존 어플리케이션, DB 중지 +docker compose -f "$WORKDIR/solid-connection-dev/docker-compose.dev.yml" down \ + || { echo "❌ 어플리케이션 도커 중지 실패: $WORKDIR/solid-connection-dev/docker-compose.dev.yml" >&2; exit 3; } +docker compose -f "$WORKDIR/mysql/docker-compose.mysql.yml" down \ + || { echo "❌ MySQL 도커 중지 실패: $WORKDIR/mysql/docker-compose.mysql.yml" >&2; exit 4; } + +# 2. 부하 테스트용 DB 실행 +docker compose -f "$WORKDIR/load-test-setting/docker-compose.load-test.yml" up -d \ + || { echo "❌ 부하 테스트용 DB 실행 실패: docker-compose.load-test.yml" >&2; exit 5; } + +# 3. MySQL 준비 대기 (최대 30초) +CONTAINER_NAME="load-test-db" +echo "⏳ MySQL이 준비될 때까지 대기 중 (최대 30초)..." +start_time=$(date +%s) +while ! docker exec "$CONTAINER_NAME" sh -c 'mysqladmin ping -h "127.0.0.1" --silent'; do + elapsed=$(( $(date +%s) - start_time )) + if [[ $elapsed -ge 30 ]]; then + echo "❌ MySQL 준비 시간 초과 (30초)" >&2 + exit 1 + fi + printf "." + sleep 1 +done +echo "✔️ MySQL 준비 완료." + +# 4. dump 주입 +echo "📥 dump 파일 복사 → 컨테이너: $CONTAINER_NAME" +docker cp "$SQL_SRC" "${CONTAINER_NAME}:/tmp/dump.sql" \ + || { echo "❌ dump 복사 실패: $SQL_SRC → $CONTAINER_NAME:/tmp/dump.sql" >&2; exit 6; } + +echo "⚙️ dump 이식" +docker exec -i "$CONTAINER_NAME" sh -c 'mysql -u root -proot < /tmp/dump.sql' \ + || { echo "❌ dump 이식 실패: 컨테이너 $CONTAINER_NAME" >&2; exit 7; } + +# 5. 어플리케이션 다시 실행 +docker compose -f "$WORKDIR/solid-connection-dev/docker-compose.dev.yml" up -d \ + || { echo "❌ 어플리케이션 재시작 실패: $WORKDIR/solid-connection-dev/docker-compose.dev.yml" >&2; exit 8; } + +echo "✅ 부하 테스트용 DB에 연결된 어플리케이션 실행 완료!" diff --git a/config/load-test/k6/set_up_xk6.sh b/config/load-test/k6/set_up_xk6.sh new file mode 100644 index 0000000..005fb3d --- /dev/null +++ b/config/load-test/k6/set_up_xk6.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -euo pipefail + +# 에러 발생 시 메시지 출력 +trap 'echo "❌ 오류 발생! 스크립트 실행이 중단되었습니다." >&2' ERR + +export GO_VERSION=1.22.2 +export BASE_DIR=/home/ubuntu/solid-connection-load-test/k6 +export GOROOT=${BASE_DIR}/go +export GOPATH=${BASE_DIR}/go-workspace +export PATH=$PATH:$GOROOT/bin:$GOPATH/bin +export XK6_BIN=${GOPATH}/bin/xk6 +export K6_OUT=xk6-prometheus-rw +export K6_PROMETHEUS_RW_SERVER_URL=http://132.145.83.182:9090/api/v1/write +export K6_PROMETHEUS_RW_TREND_STATS="p(90),p(95),p(99),avg,min,max" +{ + echo "export BASE_DIR=${BASE_DIR}" + echo "export GOROOT=${GOROOT}" + echo "export GOPATH=${GOPATH}" + echo "export PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin" + echo "export XK6_BIN=${GOPATH}/bin/xk6" + echo "export K6_OUT=xk6-prometheus-rw" + echo "export K6_PROMETHEUS_RW_SERVER_URL=http://146.56.46.8:9090/api/v1/write" + echo "K6_PROMETHEUS_RW_TREND_STATS=\"p(90),p(95),p(99),avg,min,max\"" +} >> ~/.bashrc + +echo "📁 디렉토리 생성 및 이동: $BASE_DIR" +mkdir -p "$BASE_DIR" +cd "$BASE_DIR" + +echo "⬇️ Go $GO_VERSION 다운로드 중..." +curl -OL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" + +echo "📦 Go 압축 해제 중..." +tar -xzf "go${GO_VERSION}.linux-amd64.tar.gz" +rm "go${GO_VERSION}.linux-amd64.tar.gz" + +echo "✅ Go 버전 확인: $(go version)" + +echo "⬇️ xk6 설치 중..." +go install go.k6.io/xk6/cmd/xk6@latest + +echo "✅ xk6 설치 완료: $XK6_BIN" +$XK6_BIN --help > /dev/null && echo "✅ xk6 실행 가능" + +echo "⚙️ Prometheus remote-write 플러그인을 포함한 K6 빌드 시작" +$XK6_BIN build --with github.com/grafana/xk6-output-prometheus-remote@latest + +echo "✅ 빌드 완료: $(pwd)/k6" +ls -lh ./k6 + +echo "🎉 설치가 성공적으로 완료되었습니다!" diff --git a/config/load-test/k6/updatePost.json b/config/load-test/k6/updatePost.json new file mode 100644 index 0000000..660f0a8 --- /dev/null +++ b/config/load-test/k6/updatePost.json @@ -0,0 +1,5 @@ +{ + "postCategory": "자유", + "title": "수강신청 어떻게 하나요?", + "content": "수강신청 방법을 잘 알겠어요." +} diff --git a/config/load-test/k6/whole-user-flow.js b/config/load-test/k6/whole-user-flow.js new file mode 100644 index 0000000..6fc1b47 --- /dev/null +++ b/config/load-test/k6/whole-user-flow.js @@ -0,0 +1,372 @@ +import http from 'k6/http'; +import { sleep, check, fail } from 'k6'; + +// KST +const now = new Date(); +const kstOffset = 9 * 60; // 분 단위 +const d = new Date(now.getTime() + kstOffset * 60 * 1000); +const kst = d.toISOString().slice(0, 16); // "yyyy-mm-ddTHH:MM" + +// "mm/dd HH:MM" +const time = (() => { + const [yyyy, mm, dd, hh, min] = kst.split(/[-T:]/); + return `${mm}/${dd} ${hh}:${min}`; +})(); + +const BASE_URL = 'https://api.stage.solid-connection.com'; +const testId = 'whole-user-flow'; + +export const options = { + scenarios: { + user_flow: { + executor: 'per-vu-iterations', // VU별 반복 + vus: 10, // VU + iterations: 10, // VU 한 명당 실행할 횟수 + maxDuration: '15m', // 여유로 잡아 두기 + }, + }, + tags: { + testid: testId, + time: time, + }, +}; + +function authHeadersWithTags(token) { + return { + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json; charset=utf-8', + }, + tags: { + ...options.tags, + time: time, + }, + }; +} + +function login() { + // __VU: 현재 VU 인덱스 + const email = `user${__VU}@example.com`; + const password = 'password'; + + const res = http.post(`${BASE_URL}/auth/email/sign-in`, JSON.stringify({ + email: email, + password: password, + }), { + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + tags: { + name: '/auth/email/sign-in', + } + }); + if (res.status !== 200) { + fail('로그인 실패'); + } + return res.json('accessToken'); +} + +// universites +function getRecommendedUniversities(auth) { + http.get(`${BASE_URL}/universities/recommend`, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/recommend', + }, +}); +} +function likeUniversity(id, auth) { + http.post(`${BASE_URL}/universities/${id}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/{id}/like', + }, + }); +} +function isLikedUniversity(id, auth) { + http.get(`${BASE_URL}/universities/${id}/like`, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/{id}/like', + }, + }); +} +function getLikedUniversities(auth) { + http.get(`${BASE_URL}/universities/like`, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/like', + }, +}); +} +function cancelLikeUniversity(id, auth) { + http.del(`${BASE_URL}/universities/${id}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/{id}/like', + }, + }); +} +function searchUniversities(params) { + return http.get(`${BASE_URL}/universities/search?${params}`, { + tags: { + name: '/universities/search?{params}', + }, +}); +} +function getDetailedUniversityInfo(id) { + http.get(`${BASE_URL}/universities/${id}`, { + tags: { + name: '/universities/{id}', + }, + }); +} + +// my +function getMyInfo(auth) { + http.get(`${BASE_URL}/my`, { + ...auth, + tags: { + ...auth.tags, + name: '/my', + }, +}); +} + +// users +function checkNicknameExists(nickname) { + http.get(`${BASE_URL}/users/exists?nickname=${nickname}`, { + tags: { + name: '/users/exists?nickname={nickname}', + }, + }); +} + +// boards +function getBoards(auth) { + http.get(`${BASE_URL}/boards`, { + ...auth, + tags: { + ...auth.tags, + name: '/boards', + }, +}); +} +function getPostsByBoard(boardCode, auth) { + http.get(`${BASE_URL}/boards/${boardCode}`, { + ...auth, + tags: { + ...auth.tags, + name: '/boards/{boardCode}', + }, + }); +} + +// posts +const createPostJson = open('./createPost.json', 'b'); +function createPost(token) { + const formData = { + postCreateRequest: http.file(createPostJson, 'post.json', 'application/json'), + }; + const res = http.post(`${BASE_URL}/posts`, formData, { + headers: { + Authorization: `Bearer ${token}` + }, + tags: { + testid: testId, + time: time, + name: '/posts' + }, + }); + return res.json('id'); +} +const updatePostJson = open('./updatePost.json', 'b'); +function updatePost(postId, token) { + const formData = { + postUpdateRequest: http.file(updatePostJson, 'post.json', 'application/json'), + }; + http.patch(`${BASE_URL}/posts/${postId}`, formData, { + headers: { + Authorization: `Bearer ${token}` + }, + tags: { + testid: testId, + time: time, + name: '/posts/{postId}' + }, + }); +} +function getPostDetail(postId, auth) { + http.get(`${BASE_URL}/posts/${postId}`, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}', + }, + }); +} +function likePost(postId, auth) { + http.post(`${BASE_URL}/posts/${postId}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}/like', + }, + }); +} +function cancelLikePost(postId, auth) { + http.del(`${BASE_URL}/posts/${postId}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}/like', + }, + }); +} +function deletePost(postId, auth) { + http.del(`${BASE_URL}/posts/${postId}`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}', + }, + }); +} + +// comments +function createComment(postId, auth) { + const res = http.post( + `${BASE_URL}/comments`, + JSON.stringify({ postId, content: '댓글', parentId: null }), + { + ...auth, + tags: { + ...auth.tags, + name: '/comments', + }, + }); + return res.json('id'); +} +function updateComment(commentId, auth) { + http.patch( + `${BASE_URL}/comments/${commentId}`, + JSON.stringify({ content: '댓글 수정' }), + { + ...auth, + tags: { + ...auth.tags, + name: '/comments/{commentId}', + }, + } + ); +} +function deleteComment(commentId, auth) { + http.del(`${BASE_URL}/comments/${commentId}`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/comments/{commentId}', + }, + }); +} + +// scores +function getLanguageTests(auth) { + return http.get(`${BASE_URL}/scores/language-tests`, { + ...auth, + tags: { + ...auth.tags, + name: '/scores/language-tests', + }, + }); +} +function getGPAs(auth) { + return http.get(`${BASE_URL}/scores/gpas`, { + ...auth, + tags: { + ...auth.tags, + name: '/scores/gpas', + }, + }); +} + +// applications +function apply(gpaScoreId, languageTestScoreId, universityId, auth) { + http.post(`${BASE_URL}/applications`, JSON.stringify({ + gpaScoreId: gpaScoreId, + languageTestScoreId: languageTestScoreId, + universityChoiceRequest: { + firstChoiceUniversityId: universityId, + secondChoiceUniversityId: null, + thirdChoiceUniversityId: null + }, + }), { + ...auth, + tags: { + ...auth.tags, + name: '/applications', + }, + }); +} + +function getCompetitors(auth) { + http.get(`${BASE_URL}/applications/competitors`, { + ...auth, + tags: { + ...auth.tags, + name: '/applications/competitors', + }, + }); +} + +export default function () { + checkNicknameExists(encodeURIComponent('닉네임')); + const token = login(); + const auth = authHeadersWithTags(token); + + + getRecommendedUniversities(auth); + + const uniSearchRes = searchUniversities(''); // 이번학기 열린 대학 중 랜덤하게 id 가져오기 + const uniList = uniSearchRes.json(); + const universityId = uniList[Math.floor(Math.random() * uniList.length)].id; + + likeUniversity(universityId, auth); + isLikedUniversity(universityId, auth); + getLikedUniversities(auth); + cancelLikeUniversity(universityId, auth); + getDetailedUniversityInfo(universityId); + + getMyInfo(auth); + + getBoards(auth); + getPostsByBoard('FREE', auth); + + const postId = createPost(token); + updatePost(postId, token); + getPostDetail(postId, auth); + likePost(postId, auth); + cancelLikePost(postId, auth); + + const commentId = createComment(postId, auth); + updateComment(commentId, auth); + deleteComment(commentId, auth); + + deletePost(postId, auth); + + const langRes = getLanguageTests(auth); + const langList = langRes.json().languageTestScoreStatusResponseList; + const languageTestScoreId = langList[0].id; + + const gpaRes = getGPAs(auth); + const gpaList = gpaRes.json().gpaScoreStatusResponseList; + const gpaScoreId = gpaList[0].id; + + apply(gpaScoreId, languageTestScoreId, universityId, auth); + getCompetitors(auth); + + sleep(1); +} diff --git a/environment/stage/main.tf b/environment/stage/main.tf index 3f3e129..8e60bb5 100644 --- a/environment/stage/main.tf +++ b/environment/stage/main.tf @@ -6,8 +6,8 @@ data "aws_vpc" "default" { module "stage_stack" { source = "../../modules/app_stack" - env_name = "stage" - vpc_id = data.aws_vpc.default.id + env_name = "stage" + vpc_id = data.aws_vpc.default.id ami_id = var.ami_id @@ -15,13 +15,13 @@ module "stage_stack" { ec2_iam_instance_profile = var.ec2_iam_instance_profile # 키페어 및 접속 허용 - key_name = var.key_name + key_name = var.key_name # 인스턴스 스펙 - instance_type = var.server_instance_type + instance_type = var.server_instance_type # RDS 미사용 (Docker container로 대체) - enable_rds = false + enable_rds = false # 보안 그룹 규칙 api_ingress_rules = var.api_ingress_rules @@ -41,4 +41,6 @@ module "stage_stack" { redis_version = var.redis_version redis_exporter_version = var.redis_exporter_version alloy_version = var.alloy_version + + enable_k6_files = true } diff --git a/modules/app_stack/ec2.tf b/modules/app_stack/ec2.tf index b49aa52..e1da678 100644 --- a/modules/app_stack/ec2.tf +++ b/modules/app_stack/ec2.tf @@ -21,6 +21,57 @@ data "cloudinit_config" "app_init" { content = file("${path.module}/../common/scripts/docker_setup.sh") filename = "1_docker_install.sh" } + + dynamic "part" { + for_each = var.enable_k6_files ? [1] : [] + + content { + content_type = "text/cloud-config" + content = yamlencode({ + bootcmd = [ + "mkdir -p ${var.k6_install_dir}/script" + ] + write_files = [ + { + path = "${var.k6_install_dir}/createPost.json" + owner = "ubuntu:ubuntu" + permissions = "0644" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/createPost.json") + }, + { + path = "${var.k6_install_dir}/updatePost.json" + owner = "ubuntu:ubuntu" + permissions = "0644" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/updatePost.json") + }, + { + path = "${var.k6_install_dir}/whole-user-flow.js" + owner = "ubuntu:ubuntu" + permissions = "0644" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/whole-user-flow.js") + }, + { + path = "${var.k6_install_dir}/set_up_xk6.sh" + owner = "ubuntu:ubuntu" + permissions = "0755" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/set_up_xk6.sh") + }, + { + path = "${var.k6_install_dir}/script/set-load-test.sh" + owner = "ubuntu:ubuntu" + permissions = "0755" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/script/set-load-test.sh") + } + ] + }) + filename = "2_k6_files.yml" + } + } } # API Server (EC2) @@ -100,9 +151,9 @@ resource "null_resource" "update_side_infra" { triggers = { script_hash = sha256(templatefile("${path.module}/scripts/side_infra_setup.sh.tftpl", { - work_dir = var.work_dir - alloy_env_name = var.alloy_env_name - alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { + work_dir = var.work_dir + alloy_env_name = var.alloy_env_name + alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { loki_ip = data.aws_instance.monitoring_server.private_ip }) redis_version = var.redis_version @@ -120,9 +171,9 @@ resource "null_resource" "update_side_infra" { provisioner "file" { content = templatefile("${path.module}/scripts/side_infra_setup.sh.tftpl", { - work_dir = var.work_dir - alloy_env_name = var.alloy_env_name - alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { + work_dir = var.work_dir + alloy_env_name = var.alloy_env_name + alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { loki_ip = data.aws_instance.monitoring_server.private_ip }) redis_version = var.redis_version diff --git a/modules/app_stack/variables.tf b/modules/app_stack/variables.tf index 33f8b1a..c1d199c 100644 --- a/modules/app_stack/variables.tf +++ b/modules/app_stack/variables.tf @@ -67,7 +67,7 @@ variable "additional_db_users" { database = string privileges = list(string) })) - default = {} + default = {} } variable "db_engine_version" { @@ -155,3 +155,15 @@ variable "alloy_version" { description = "Docker image tag for Grafana Alloy" type = string } + +variable "enable_k6_files" { + description = "Whether to place k6 load test files on the API server during cloud-init" + type = bool + default = false +} + +variable "k6_install_dir" { + description = "Directory where k6 load test files are placed" + type = string + default = "/home/ubuntu/solid-connection-load-test/k6" +} diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 0283669..f929c01 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -1,61 +1,74 @@ -# Load Test Automation +# 부하 테스트 자동화 -This automation creates a temporary load test RDS instance, copies prod RDS data -into it, writes load test datasource values to Parameter Store, and switches the -stage application through SSM Run Command. +부하 테스트용 임시 RDS를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 +loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작과 종료는 GitHub +Actions에서 수동으로 실행합니다. -## GitHub Actions Flow +## 시작 -### Start +1. GitHub에서 **Actions > Load Test Start**를 엽니다. +2. **Run workflow**를 클릭합니다. +3. 기본값 그대로 실행합니다. -1. Open **Actions > Load Test Start**. -2. Click **Run workflow**. -3. Keep `switch_stage_to_loadtest` enabled to restart stage with - `dev,loadtest` profiles. -4. Keep `copy_prod_data` enabled to copy prod RDS data into the load test RDS. +입력값: -The workflow runs `scripts/load_test/start.sh`. +- `switch_stage_to_loadtest`: stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +- `copy_prod_data`: prod RDS 데이터를 loadtest RDS로 복사합니다. -### Stop +시작 workflow는 `scripts/load_test/start.sh`를 실행합니다. -1. Open **Actions > Load Test Stop**. -2. Click **Run workflow**. -3. Keep `restore_stage_dev` enabled to restart stage with the normal dev - compose configuration. -4. Keep `destroy_rds` enabled to destroy the load test Terraform stack. +## 종료 -The workflow runs `scripts/load_test/stop.sh`. +1. GitHub에서 **Actions > Load Test Stop**을 엽니다. +2. **Run workflow**를 클릭합니다. +3. 기본값 그대로 실행합니다. -## What Start Does +입력값: -1. Runs `terraform apply` in `environment/load_test`. -2. Creates the load test RDS and writes: +- `restore_stage_dev`: stage 앱을 기존 dev compose 구성으로 되돌립니다. +- `destroy_rds`: loadtest Terraform stack을 제거합니다. + +종료 workflow는 `scripts/load_test/stop.sh`를 실행합니다. + +## 시작 시 수행 작업 + +1. `environment/load_test`에서 `terraform apply`를 실행합니다. +2. loadtest RDS를 생성하고 아래 Parameter Store 값을 작성합니다. - `/solid-connection/loadtest/spring.datasource.url` - `/solid-connection/loadtest/spring.datasource.username` - `/solid-connection/loadtest/spring.datasource.password` -3. Switches the stage app to `dev,loadtest` profiles through SSM Run Command. -4. Stores DB migration credentials in temporary SSM parameters. -5. Runs `mysqldump` on the prod EC2 instance through SSM Run Command and - restores the dump into the load test RDS. -6. Deletes the temporary migration parameters. - -## What Stop Does - -1. Restores the stage app to the normal dev compose configuration through SSM - Run Command. -2. Runs `terraform destroy` for the load test Terraform stack. - -## Notes - -- GitHub Actions uses `AWS_ROLE_ARN` through OIDC. -- Private submodule checkout uses `GH_PAT`. -- No SSH private key is required for load test start/stop. -- The prod and stage EC2 instances are looked up by their `Name` tags. -- Prod DB username/password are read from Parameter Store. The default paths are - `/solid-connection/prod/spring.datasource.username` and - `/solid-connection/prod/spring.datasource.password`. -- Load test DB username/password are read from Parameter Store. The default paths - are `/solid-connection/loadtest/spring.datasource.username` and - `/solid-connection/loadtest/spring.datasource.password`. -- The load test RDS security group allows MySQL only from the security groups - attached to the prod and stage API EC2 instances. +3. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +4. migration용 임시 Parameter Store 값을 생성합니다. +5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS에 복원합니다. +6. migration용 임시 Parameter Store 값을 삭제합니다. + +## 종료 시 수행 작업 + +1. SSM RunCommand로 stage 앱을 기존 dev compose 구성으로 되돌립니다. +2. `environment/load_test`에서 `terraform destroy`를 실행합니다. + +## k6 파일 + +stage EC2를 새로 생성하는 경우 Terraform cloud-init이 +`/home/ubuntu/solid-connection-load-test/k6`에 k6 파일을 배치합니다. + +현재 포함된 파일: + +- `createPost.json` +- `updatePost.json` +- `whole-user-flow.js` +- `set_up_xk6.sh` +- `script/set-load-test.sh` + +기존 stage EC2는 재생성하지 않으므로 이 cloud-init 변경이 즉시 반영되지는 않습니다. + +## 참고 사항 + +- GitHub Actions는 OIDC로 `AWS_ROLE_ARN`을 assume합니다. +- private submodule checkout에는 `GH_PAT`를 사용합니다. +- SSH private key는 사용하지 않습니다. +- prod/stage EC2는 `Name` tag로 조회합니다. +- prod DB username/password는 Parameter Store에서 읽습니다. +- loadtest DB username/password도 Parameter Store에서 읽습니다. +- loadtest RDS security group은 prod/stage API EC2 security group에서 오는 MySQL + 접근만 허용합니다. From 7d59011e68aa77b0197d006a5b3dc1c50dc62ce9 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 14:33:31 +0900 Subject: [PATCH 06/17] =?UTF-8?q?fix:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=9E=90=EB=8F=99=ED=99=94=20=EB=A6=AC?= =?UTF-8?q?=EB=B7=B0=20=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: load_test Terraform plan workflow를 추가했습니다. - 상세내용: loadtest RDS 네트워크를 stage EC2 VPC 기준으로 생성하도록 수정했습니다. - 상세내용: SSM 명령 timeout, dump cleanup, k6 파일 동기화, 데이터 복사 후 stage 전환 순서를 반영했습니다. - 상세내용: k6 설정과 응답 검증 오류를 수정했습니다. --- .github/workflows/terraform-plan.yml | 70 ++++++++++++- config/load-test/k6/set_up_xk6.sh | 30 +++--- config/load-test/k6/whole-user-flow.js | 28 +++-- environment/load_test/main.tf | 26 ++--- environment/load_test/output.tf | 20 ++++ environment/load_test/variables.tf | 12 +++ scripts/load_test/README.md | 69 ++++++------ scripts/load_test/start.sh | 140 +++++++++++++++++++------ scripts/load_test/stop.sh | 14 +++ 9 files changed, 309 insertions(+), 100 deletions(-) diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml index 870f6e3..5f5d1af 100644 --- a/.github/workflows/terraform-plan.yml +++ b/.github/workflows/terraform-plan.yml @@ -21,6 +21,7 @@ jobs: global: ${{ steps.filter.outputs.global }} prod: ${{ steps.filter.outputs.prod }} stage: ${{ steps.filter.outputs.stage }} + load_test: ${{ steps.filter.outputs.load_test }} monitoring: ${{ steps.filter.outputs.monitoring }} steps: - uses: actions/checkout@v4 @@ -49,6 +50,9 @@ jobs: - 'modules/common/**' - 'config/secrets/stage.tfvars' - 'config/secrets/app_stack.tfvars' + load_test: + - 'environment/load_test/**' + - 'config/secrets/load_test.tfvars' monitoring: - 'environment/monitoring/**' - 'modules/monitoring_stack/**' @@ -373,6 +377,69 @@ jobs: if: steps.plan.outputs.exitcode == '1' run: exit 1 + plan-load_test: + needs: detect-changes + if: needs.detect-changes.outputs.load_test == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_PLAN_ROLE_ARN }} + aws-region: ap-northeast-2 + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + - name: Terraform Init + working-directory: environment/load_test + run: terraform init + - name: Terraform Plan + id: plan + working-directory: environment/load_test + run: | + terraform plan -no-color \ + -var-file="../../config/secrets/load_test.tfvars" \ + 2>&1 | tee plan_output.txt + echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT + - name: Upload Plan Artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: terraform-plan-load-test + path: environment/load_test/plan_output.txt + - name: Post Plan Comment + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const marker = ''; + const planFile = 'environment/load_test/plan_output.txt'; + const output = fs.existsSync(planFile) ? fs.readFileSync(planFile, 'utf8') : ''; + const summary = output.split('\n').find(l => /^Plan:/.test(l)) || output.split('\n').find(l => /No changes/.test(l)) || (output ? '(plan result parse failed)' : 'plan failed before writing output. Check workflow logs.'); + const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = `${marker}\n## Terraform Plan: \`load_test\`\n\n${summary}\n\n> Full plan output is kept in the workflow artifact for security. Check [workflow run artifact](${runUrl}).`; + + const { data: comments } = await github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + }); + const existing = comments.find(c => c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ comment_id: existing.id, owner: context.repo.owner, repo: context.repo.repo, body }); + } else { + await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body }); + } + - name: Plan Status Check + if: steps.plan.outputs.exitcode == '1' + run: exit 1 + plan-monitoring: needs: detect-changes if: needs.detect-changes.outputs.monitoring == 'true' @@ -438,7 +505,7 @@ jobs: run: exit 1 trigger-coderabbit: - needs: [plan-bootstrap, plan-global, plan-prod, plan-stage, plan-monitoring] + needs: [plan-bootstrap, plan-global, plan-prod, plan-stage, plan-load_test, plan-monitoring] if: | always() && ( @@ -446,6 +513,7 @@ jobs: needs.plan-global.result == 'success' || needs.plan-global.result == 'failure' || needs.plan-prod.result == 'success' || needs.plan-prod.result == 'failure' || needs.plan-stage.result == 'success' || needs.plan-stage.result == 'failure' || + needs.plan-load_test.result == 'success' || needs.plan-load_test.result == 'failure' || needs.plan-monitoring.result == 'success' || needs.plan-monitoring.result == 'failure' ) runs-on: ubuntu-latest diff --git a/config/load-test/k6/set_up_xk6.sh b/config/load-test/k6/set_up_xk6.sh index 005fb3d..01ad6d6 100644 --- a/config/load-test/k6/set_up_xk6.sh +++ b/config/load-test/k6/set_up_xk6.sh @@ -2,8 +2,7 @@ set -euo pipefail -# 에러 발생 시 메시지 출력 -trap 'echo "❌ 오류 발생! 스크립트 실행이 중단되었습니다." >&2' ERR +trap 'echo "xk6 setup failed" >&2' ERR export GO_VERSION=1.22.2 export BASE_DIR=/home/ubuntu/solid-connection-load-test/k6 @@ -14,6 +13,7 @@ export XK6_BIN=${GOPATH}/bin/xk6 export K6_OUT=xk6-prometheus-rw export K6_PROMETHEUS_RW_SERVER_URL=http://132.145.83.182:9090/api/v1/write export K6_PROMETHEUS_RW_TREND_STATS="p(90),p(95),p(99),avg,min,max" + { echo "export BASE_DIR=${BASE_DIR}" echo "export GOROOT=${GOROOT}" @@ -21,33 +21,33 @@ export K6_PROMETHEUS_RW_TREND_STATS="p(90),p(95),p(99),avg,min,max" echo "export PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin" echo "export XK6_BIN=${GOPATH}/bin/xk6" echo "export K6_OUT=xk6-prometheus-rw" - echo "export K6_PROMETHEUS_RW_SERVER_URL=http://146.56.46.8:9090/api/v1/write" - echo "K6_PROMETHEUS_RW_TREND_STATS=\"p(90),p(95),p(99),avg,min,max\"" + echo "export K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL}" + echo "export K6_PROMETHEUS_RW_TREND_STATS=\"${K6_PROMETHEUS_RW_TREND_STATS}\"" } >> ~/.bashrc -echo "📁 디렉토리 생성 및 이동: $BASE_DIR" +echo "Create and enter ${BASE_DIR}" mkdir -p "$BASE_DIR" cd "$BASE_DIR" -echo "⬇️ Go $GO_VERSION 다운로드 중..." +echo "Download Go ${GO_VERSION}" curl -OL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" -echo "📦 Go 압축 해제 중..." +echo "Extract Go" tar -xzf "go${GO_VERSION}.linux-amd64.tar.gz" rm "go${GO_VERSION}.linux-amd64.tar.gz" -echo "✅ Go 버전 확인: $(go version)" +echo "Go version: $(go version)" -echo "⬇️ xk6 설치 중..." +echo "Install xk6" go install go.k6.io/xk6/cmd/xk6@latest -echo "✅ xk6 설치 완료: $XK6_BIN" -$XK6_BIN --help > /dev/null && echo "✅ xk6 실행 가능" +echo "xk6 installed: ${XK6_BIN}" +"$XK6_BIN" --help > /dev/null && echo "xk6 executable is available" -echo "⚙️ Prometheus remote-write 플러그인을 포함한 K6 빌드 시작" -$XK6_BIN build --with github.com/grafana/xk6-output-prometheus-remote@latest +echo "Build k6 with Prometheus remote-write output" +"$XK6_BIN" build --with github.com/grafana/xk6-output-prometheus-remote@latest -echo "✅ 빌드 완료: $(pwd)/k6" +echo "Build complete: $(pwd)/k6" ls -lh ./k6 -echo "🎉 설치가 성공적으로 완료되었습니다!" +echo "xk6 setup completed" diff --git a/config/load-test/k6/whole-user-flow.js b/config/load-test/k6/whole-user-flow.js index 6fc1b47..a40f82c 100644 --- a/config/load-test/k6/whole-user-flow.js +++ b/config/load-test/k6/whole-user-flow.js @@ -293,6 +293,20 @@ function getGPAs(auth) { }); } +function requireArray(value, name) { + if (!Array.isArray(value) || value.length === 0) { + fail(`${name} response is empty or invalid`); + } + return value; +} + +function requireId(value, name) { + if (!value || value.id === undefined || value.id === null) { + fail(`${name} response does not contain id`); + } + return value.id; +} + // applications function apply(gpaScoreId, languageTestScoreId, universityId, auth) { http.post(`${BASE_URL}/applications`, JSON.stringify({ @@ -323,7 +337,7 @@ function getCompetitors(auth) { } export default function () { - checkNicknameExists(encodeURIComponent('닉네임')); + checkNicknameExists(encodeURIComponent('loadtest-user')); const token = login(); const auth = authHeadersWithTags(token); @@ -331,8 +345,8 @@ export default function () { getRecommendedUniversities(auth); const uniSearchRes = searchUniversities(''); // 이번학기 열린 대학 중 랜덤하게 id 가져오기 - const uniList = uniSearchRes.json(); - const universityId = uniList[Math.floor(Math.random() * uniList.length)].id; + const uniList = requireArray(uniSearchRes.json(), 'universities/search'); + const universityId = requireId(uniList[Math.floor(Math.random() * uniList.length)], 'universities/search item'); likeUniversity(universityId, auth); isLikedUniversity(universityId, auth); @@ -358,12 +372,12 @@ export default function () { deletePost(postId, auth); const langRes = getLanguageTests(auth); - const langList = langRes.json().languageTestScoreStatusResponseList; - const languageTestScoreId = langList[0].id; + const langList = requireArray(langRes.json().languageTestScoreStatusResponseList, 'scores/language-tests'); + const languageTestScoreId = requireId(langList[0], 'scores/language-tests item'); const gpaRes = getGPAs(auth); - const gpaList = gpaRes.json().gpaScoreStatusResponseList; - const gpaScoreId = gpaList[0].id; + const gpaList = requireArray(gpaRes.json().gpaScoreStatusResponseList, 'scores/gpas'); + const gpaScoreId = requireId(gpaList[0], 'scores/gpas item'); apply(gpaScoreId, languageTestScoreId, universityId, auth); getCompetitors(auth); diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index 72b1c58..2b5716f 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -1,14 +1,3 @@ -data "aws_vpc" "default" { - default = true -} - -data "aws_subnets" "default" { - filter { - name = "vpc-id" - values = [data.aws_vpc.default.id] - } -} - data "aws_instance" "prod_api" { filter { name = "tag:Name" @@ -33,6 +22,17 @@ data "aws_instance" "stage_api" { } } +data "aws_subnet" "stage_api" { + id = data.aws_instance.stage_api.subnet_id +} + +data "aws_subnets" "target" { + filter { + name = "vpc-id" + values = [data.aws_subnet.stage_api.vpc_id] + } +} + data "aws_db_instance" "prod" { db_instance_identifier = var.prod_rds_identifier } @@ -59,7 +59,7 @@ locals { resource "aws_security_group" "load_test_db" { name = "sc-load-test-db-sg" description = "Security group for load test RDS" - vpc_id = data.aws_vpc.default.id + vpc_id = data.aws_subnet.stage_api.vpc_id egress { from_port = 0 @@ -87,7 +87,7 @@ resource "aws_security_group_rule" "load_test_db_mysql" { resource "aws_db_subnet_group" "load_test" { name = "sc-load-test-db-subnet-group" - subnet_ids = data.aws_subnets.default.ids + subnet_ids = data.aws_subnets.target.ids tags = { Name = "solid-connection-load-test-db-subnet-group" diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf index 55390ac..ee0bc25 100644 --- a/environment/load_test/output.tf +++ b/environment/load_test/output.tf @@ -47,3 +47,23 @@ output "load_test_ssm_parameter_prefix" { description = "SSM Parameter Store prefix for load test datasource values" value = var.load_test_parameter_prefix } + +output "load_test_db_username_parameter_name" { + description = "SSM parameter name containing the load test DB username" + value = var.load_test_db_username_parameter_name +} + +output "load_test_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the load test DB password" + value = var.load_test_db_password_parameter_name +} + +output "prod_db_username_parameter_name" { + description = "SSM parameter name containing the prod DB username" + value = var.prod_db_username_parameter_name +} + +output "prod_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the prod DB password" + value = var.prod_db_password_parameter_name +} diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index 7d4d639..efdfa28 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -40,6 +40,18 @@ variable "load_test_db_password_parameter_name" { type = string } +variable "prod_db_username_parameter_name" { + description = "SSM parameter name containing the prod DB username" + type = string + default = "/solid-connection/prod/spring.datasource.username" +} + +variable "prod_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the prod DB password" + type = string + default = "/solid-connection/prod/spring.datasource.password" +} + variable "kms_key_arn" { description = "KMS key ARN for RDS storage encryption" type = string diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index f929c01..3647c44 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -1,58 +1,56 @@ # 부하 테스트 자동화 -부하 테스트용 임시 RDS를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 -loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작과 종료는 GitHub -Actions에서 수동으로 실행합니다. +부하 테스트용 임시 RDS를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작과 종료는 GitHub Actions에서 수동으로 실행합니다. + +## 원칙 + +- 사람이 로컬에서 `terraform apply` 또는 `terraform destroy`를 직접 실행하지 않습니다. +- 시작은 **Actions > Load Test Start** workflow로 실행합니다. +- 종료는 **Actions > Load Test Stop** workflow로 실행합니다. +- SSH private key는 사용하지 않습니다. stage/prod EC2 작업은 SSM RunCommand로 실행합니다. ## 시작 1. GitHub에서 **Actions > Load Test Start**를 엽니다. 2. **Run workflow**를 클릭합니다. -3. 기본값 그대로 실행합니다. +3. 필요한 입력값을 선택하고 실행합니다. 입력값: - `switch_stage_to_loadtest`: stage 앱을 `dev,loadtest` 프로필로 재기동합니다. - `copy_prod_data`: prod RDS 데이터를 loadtest RDS로 복사합니다. -시작 workflow는 `scripts/load_test/start.sh`를 실행합니다. +workflow는 GitHub Actions runner에서 `environment/load_test`의 Terraform을 apply하고 `scripts/load_test/start.sh`를 실행합니다. + +## 시작 시 수행 작업 + +1. GitHub Actions가 `environment/load_test`에서 Terraform apply를 실행합니다. +2. loadtest RDS와 보안 그룹을 생성합니다. RDS는 stage EC2가 속한 VPC/subnet 기준으로 생성됩니다. +3. loadtest datasource 값을 Parameter Store에 작성합니다. +4. prod DB와 loadtest DB 접속 정보를 Parameter Store에서 읽습니다. +5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS로 복원합니다. +6. stage EC2에 k6 파일을 `/home/ubuntu/solid-connection-load-test/k6` 경로로 동기화합니다. +7. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +8. 데이터 이관용 임시 Parameter Store 값을 삭제합니다. ## 종료 1. GitHub에서 **Actions > Load Test Stop**을 엽니다. 2. **Run workflow**를 클릭합니다. -3. 기본값 그대로 실행합니다. +3. 필요한 입력값을 선택하고 실행합니다. 입력값: - `restore_stage_dev`: stage 앱을 기존 dev compose 구성으로 되돌립니다. - `destroy_rds`: loadtest Terraform stack을 제거합니다. -종료 workflow는 `scripts/load_test/stop.sh`를 실행합니다. - -## 시작 시 수행 작업 - -1. `environment/load_test`에서 `terraform apply`를 실행합니다. -2. loadtest RDS를 생성하고 아래 Parameter Store 값을 작성합니다. - - `/solid-connection/loadtest/spring.datasource.url` - - `/solid-connection/loadtest/spring.datasource.username` - - `/solid-connection/loadtest/spring.datasource.password` -3. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. -4. migration용 임시 Parameter Store 값을 생성합니다. -5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS에 복원합니다. -6. migration용 임시 Parameter Store 값을 삭제합니다. - -## 종료 시 수행 작업 - -1. SSM RunCommand로 stage 앱을 기존 dev compose 구성으로 되돌립니다. -2. `environment/load_test`에서 `terraform destroy`를 실행합니다. +workflow는 `scripts/load_test/stop.sh`를 실행하고, 선택값에 따라 stage 복구와 Terraform destroy를 수행합니다. ## k6 파일 -stage EC2를 새로 생성하는 경우 Terraform cloud-init이 -`/home/ubuntu/solid-connection-load-test/k6`에 k6 파일을 배치합니다. +stage EC2를 새로 생성하는 경우 Terraform cloud-init이 k6 파일을 배치합니다. 기존 stage EC2는 cloud-init이 다시 실행되지 않으므로, **Load Test Start** workflow가 실행될 때 SSM으로 k6 파일을 다시 동기화합니다. -현재 포함된 파일: +포함 파일: - `createPost.json` - `updatePost.json` @@ -60,15 +58,18 @@ stage EC2를 새로 생성하는 경우 Terraform cloud-init이 - `set_up_xk6.sh` - `script/set-load-test.sh` -기존 stage EC2는 재생성하지 않으므로 이 cloud-init 변경이 즉시 반영되지는 않습니다. +stage EC2에 접속해 수동으로 실행해야 한다면 다음 경로에서 실행합니다. + +```bash +cd /home/ubuntu/solid-connection-load-test/k6 +./set_up_xk6.sh +./script/set-load-test.sh +``` -## 참고 사항 +## 참고 - GitHub Actions는 OIDC로 `AWS_ROLE_ARN`을 assume합니다. - private submodule checkout에는 `GH_PAT`를 사용합니다. -- SSH private key는 사용하지 않습니다. - prod/stage EC2는 `Name` tag로 조회합니다. -- prod DB username/password는 Parameter Store에서 읽습니다. -- loadtest DB username/password도 Parameter Store에서 읽습니다. -- loadtest RDS security group은 prod/stage API EC2 security group에서 오는 MySQL - 접근만 허용합니다. +- prod/loadtest DB username/password는 Parameter Store에서 읽습니다. +- loadtest RDS security group은 prod/stage API EC2 security group에서 오는 MySQL 접근만 허용합니다. diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh index 1f9c75a..21ab43d 100644 --- a/scripts/load_test/start.sh +++ b/scripts/load_test/start.sh @@ -3,15 +3,18 @@ set -euo pipefail TERRAFORM_DIR="environment/load_test" VAR_FILE="../../config/secrets/load_test.tfvars" -DATABASE_NAME="solid_connection" +DATABASE_NAME="" MIGRATION_PARAMETER_PREFIX="/solid-connection/loadtest/migration" -PROD_DB_USERNAME_PARAMETER="/solid-connection/prod/spring.datasource.username" -PROD_DB_PASSWORD_PARAMETER="/solid-connection/prod/spring.datasource.password" -LOADTEST_DB_USERNAME_PARAMETER="/solid-connection/loadtest/spring.datasource.username" -LOADTEST_DB_PASSWORD_PARAMETER="/solid-connection/loadtest/spring.datasource.password" +PROD_DB_USERNAME_PARAMETER="" +PROD_DB_PASSWORD_PARAMETER="" +LOADTEST_DB_USERNAME_PARAMETER="" +LOADTEST_DB_PASSWORD_PARAMETER="" SWITCH_STAGE_TO_LOADTEST="false" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" +STAGE_K6_DIR="/home/ubuntu/solid-connection-load-test/k6" +LOCAL_K6_DIR="config/load-test/k6" +SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-1800}" SKIP_TERRAFORM_APPLY="false" SKIP_DATA_COPY="false" @@ -22,15 +25,18 @@ Usage: scripts/load_test/start.sh [options] Options: --terraform-dir PATH Default: environment/load_test --var-file PATH Default: ../../config/secrets/load_test.tfvars - --prod-db-username-parameter Default: /solid-connection/prod/spring.datasource.username - --prod-db-password-parameter Default: /solid-connection/prod/spring.datasource.password - --loadtest-db-username-parameter Default: /solid-connection/loadtest/spring.datasource.username - --loadtest-db-password-parameter Default: /solid-connection/loadtest/spring.datasource.password - --database-name VALUE Default: solid_connection + --prod-db-username-parameter Default: Terraform output prod_db_username_parameter_name + --prod-db-password-parameter Default: Terraform output prod_db_password_parameter_name + --loadtest-db-username-parameter Default: Terraform output load_test_db_username_parameter_name + --loadtest-db-password-parameter Default: Terraform output load_test_db_password_parameter_name + --database-name VALUE Default: Terraform output load_test_db_name --migration-prefix VALUE Default: /solid-connection/loadtest/migration --switch-stage-to-loadtest Restart stage app through SSM with dev,loadtest profiles --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml + --stage-k6-dir PATH Default: /home/ubuntu/solid-connection-load-test/k6 + --local-k6-dir PATH Default: config/load-test/k6 + --ssm-command-timeout-seconds Default: 1800 --skip-terraform-apply --skip-data-copy -h, --help @@ -50,6 +56,9 @@ while [[ $# -gt 0 ]]; do --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --stage-k6-dir) STAGE_K6_DIR="$2"; shift 2 ;; + --local-k6-dir) LOCAL_K6_DIR="$2"; shift 2 ;; + --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; --skip-data-copy) SKIP_DATA_COPY="true"; shift ;; -h|--help) usage; exit 0 ;; @@ -76,6 +85,7 @@ require_command() { require_command terraform require_command aws require_command jq +require_command base64 tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" @@ -87,6 +97,7 @@ send_ssm_command() { local commands_json="$3" local command_id + local started_at command_id="$(aws ssm send-command \ --instance-ids "$instance_id" \ --document-name "AWS-RunShellScript" \ @@ -94,6 +105,7 @@ send_ssm_command() { --parameters "$commands_json" \ --query "Command.CommandId" \ --output text)" + started_at="$(date +%s)" local status while true; do @@ -104,6 +116,15 @@ send_ssm_command() { --query "Status" \ --output text 2>/dev/null || true)" + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command timed out after ${SSM_COMMAND_TIMEOUT_SECONDS}s: $comment" >&2 + exit 1 + fi + case "$status" in Pending|InProgress|Delayed|"") continue ;; Success) break ;; @@ -119,6 +140,51 @@ send_ssm_command() { done } +file_base64() { + base64 "$1" | tr -d '\n' +} + +sync_stage_k6_files() { + local instance_id="$1" + local commands + commands="$(jq -cn \ + --arg target_dir "$STAGE_K6_DIR" \ + '{ commands: ["set -euo pipefail", "mkdir -p \($target_dir)/script"] }')" + + local relative_path + for relative_path in \ + "createPost.json" \ + "updatePost.json" \ + "whole-user-flow.js" \ + "set_up_xk6.sh" \ + "script/set-load-test.sh"; do + local source_path="${LOCAL_K6_DIR}/${relative_path}" + if [[ ! -f "$source_path" ]]; then + echo "Missing k6 file: $source_path" >&2 + exit 1 + fi + + commands="$(jq -cn \ + --argjson current "$commands" \ + --arg target "${STAGE_K6_DIR}/${relative_path}" \ + --arg content "$(file_base64 "$source_path")" \ + '$current | .commands += [ + "mkdir -p \"$(dirname \"\($target)\")\"", + "printf %s \($content | @sh) | base64 -d > \($target | @sh)" + ]')" + done + + commands="$(jq -cn \ + --argjson current "$commands" \ + --arg target_dir "$STAGE_K6_DIR" \ + '$current | .commands += [ + "chmod +x \($target_dir)/set_up_xk6.sh \($target_dir)/script/set-load-test.sh", + "chown -R ubuntu:ubuntu \($target_dir)" + ]')" + + send_ssm_command "$instance_id" "Sync k6 files to stage EC2" "$commands" +} + delete_temp_parameters() { aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" >/dev/null 2>&1 || true aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" >/dev/null 2>&1 || true @@ -138,27 +204,17 @@ prod_endpoint="$(tf_output prod_rds_endpoint)" prod_port="$(tf_output prod_rds_port)" loadtest_endpoint="$(tf_output load_test_rds_endpoint)" loadtest_port="$(tf_output load_test_rds_port)" +loadtest_db_name="$(tf_output load_test_db_name)" +tf_prod_db_username_parameter="$(tf_output prod_db_username_parameter_name)" +tf_prod_db_password_parameter="$(tf_output prod_db_password_parameter_name)" +tf_loadtest_db_username_parameter="$(tf_output load_test_db_username_parameter_name)" +tf_loadtest_db_password_parameter="$(tf_output load_test_db_password_parameter_name)" -if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then - stage_commands_json="$(jq -cn \ - --arg app_dir "$STAGE_APP_DIR" \ - --arg compose_file "$STAGE_COMPOSE_FILE" \ - '{ - commands: [ - "set -euo pipefail", - "cd \($app_dir)", - "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", - "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", - "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", - "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", - "cat > docker-compose.loadtest.override.yml <<'\''YAML'\''\nservices:\n solid-connection-dev:\n environment:\n - SPRING_PROFILES_ACTIVE=dev,loadtest\n - AWS_REGION=ap-northeast-2\n - SPRING_DATA_REDIS_HOST=127.0.0.1\n - SPRING_DATA_REDIS_PORT=6379\nYAML", - "docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml down || true", - "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml up -d solid-connection-dev" - ] - }')" - - send_ssm_command "$stage_instance_id" "Switch stage app to load test datasource" "$stage_commands_json" -fi +DATABASE_NAME="${DATABASE_NAME:-$loadtest_db_name}" +PROD_DB_USERNAME_PARAMETER="${PROD_DB_USERNAME_PARAMETER:-$tf_prod_db_username_parameter}" +PROD_DB_PASSWORD_PARAMETER="${PROD_DB_PASSWORD_PARAMETER:-$tf_prod_db_password_parameter}" +LOADTEST_DB_USERNAME_PARAMETER="${LOADTEST_DB_USERNAME_PARAMETER:-$tf_loadtest_db_username_parameter}" +LOADTEST_DB_PASSWORD_PARAMETER="${LOADTEST_DB_PASSWORD_PARAMETER:-$tf_loadtest_db_password_parameter}" if [[ "$SKIP_DATA_COPY" != "true" ]]; then trap delete_temp_parameters EXIT @@ -226,6 +282,7 @@ if [[ "$SKIP_DATA_COPY" != "true" ]]; then "LOAD_USER=$(aws ssm get-parameter --name \($prefix)/loadtest-db-username --query Parameter.Value --output text)", "LOAD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/loadtest-db-password --with-decryption --query Parameter.Value --output text)", "DUMP_FILE=/tmp/solid-connection-loadtest-$(date +%Y%m%d%H%M%S).sql.gz", + "trap '\''rm -f \"$DUMP_FILE\"'\'' EXIT", "MYSQL_PWD=\"$PROD_PASSWORD\" mysqldump --single-transaction --set-gtid-purged=OFF --column-statistics=0 -h \($prod_endpoint) -P \($prod_port) -u \"$PROD_USER\" \($database) | gzip > \"$DUMP_FILE\"", "MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" -e \"DROP DATABASE IF EXISTS \\\`\($database)\\\`; CREATE DATABASE \\\`\($database)\\\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;\"", "gunzip -c \"$DUMP_FILE\" | MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" \($database)", @@ -236,6 +293,29 @@ if [[ "$SKIP_DATA_COPY" != "true" ]]; then send_ssm_command "$prod_instance_id" "Copy prod RDS data to load test RDS" "$copy_commands_json" fi +if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then + sync_stage_k6_files "$stage_instance_id" + + stage_commands_json="$(jq -cn \ + --arg app_dir "$STAGE_APP_DIR" \ + --arg compose_file "$STAGE_COMPOSE_FILE" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($app_dir)", + "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", + "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", + "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", + "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", + "cat > docker-compose.loadtest.override.yml <<'\''YAML'\''\nservices:\n solid-connection-dev:\n environment:\n - SPRING_PROFILES_ACTIVE=dev,loadtest\n - AWS_REGION=ap-northeast-2\n - SPRING_DATA_REDIS_HOST=127.0.0.1\n - SPRING_DATA_REDIS_PORT=6379\nYAML", + "docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml down || true", + "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml up -d solid-connection-dev" + ] + }')" + + send_ssm_command "$stage_instance_id" "Switch stage app to load test datasource" "$stage_commands_json" +fi + echo "Load test environment is ready." echo "RDS endpoint: ${loadtest_endpoint}:${loadtest_port}" echo "Stage instance: ${stage_instance_id}" diff --git a/scripts/load_test/stop.sh b/scripts/load_test/stop.sh index 62687fb..005a61c 100644 --- a/scripts/load_test/stop.sh +++ b/scripts/load_test/stop.sh @@ -6,6 +6,7 @@ VAR_FILE="../../config/secrets/load_test.tfvars" RESTORE_STAGE_DEV="false" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" +SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-900}" SKIP_TERRAFORM_DESTROY="false" usage() { @@ -18,6 +19,7 @@ Options: --restore-stage-dev Restart stage app through SSM with dev profile --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml + --ssm-command-timeout-seconds Default: 900 --skip-terraform-destroy -h, --help EOF @@ -30,6 +32,7 @@ while [[ $# -gt 0 ]]; do --restore-stage-dev) RESTORE_STAGE_DEV="true"; shift ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; --skip-terraform-destroy) SKIP_TERRAFORM_DESTROY="true"; shift ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1" >&2; usage; exit 1 ;; @@ -57,6 +60,7 @@ send_ssm_command() { local commands_json="$3" local command_id + local started_at command_id="$(aws ssm send-command \ --instance-ids "$instance_id" \ --document-name "AWS-RunShellScript" \ @@ -64,6 +68,7 @@ send_ssm_command() { --parameters "$commands_json" \ --query "Command.CommandId" \ --output text)" + started_at="$(date +%s)" local status while true; do @@ -74,6 +79,15 @@ send_ssm_command() { --query "Status" \ --output text 2>/dev/null || true)" + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command timed out after ${SSM_COMMAND_TIMEOUT_SECONDS}s: $comment" >&2 + exit 1 + fi + case "$status" in Pending|InProgress|Delayed|"") continue ;; Success) break ;; From 4878107fd7e813b788e4baa60703ce8466e8b8ad Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:04:55 +0900 Subject: [PATCH 07/17] =?UTF-8?q?chore:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20plan=20=EC=9E=90=EB=8F=99=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20=EC=A0=9C=EC=99=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: 임시로 생성되는 load_test 환경을 PR Terraform plan 대상에서 제외했습니다. - 상세내용: load_test apply와 destroy는 수동 GitHub Actions workflow에서만 실행하도록 정리했습니다. --- .github/workflows/terraform-plan.yml | 70 +--------------------------- 1 file changed, 1 insertion(+), 69 deletions(-) diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml index 5f5d1af..870f6e3 100644 --- a/.github/workflows/terraform-plan.yml +++ b/.github/workflows/terraform-plan.yml @@ -21,7 +21,6 @@ jobs: global: ${{ steps.filter.outputs.global }} prod: ${{ steps.filter.outputs.prod }} stage: ${{ steps.filter.outputs.stage }} - load_test: ${{ steps.filter.outputs.load_test }} monitoring: ${{ steps.filter.outputs.monitoring }} steps: - uses: actions/checkout@v4 @@ -50,9 +49,6 @@ jobs: - 'modules/common/**' - 'config/secrets/stage.tfvars' - 'config/secrets/app_stack.tfvars' - load_test: - - 'environment/load_test/**' - - 'config/secrets/load_test.tfvars' monitoring: - 'environment/monitoring/**' - 'modules/monitoring_stack/**' @@ -377,69 +373,6 @@ jobs: if: steps.plan.outputs.exitcode == '1' run: exit 1 - plan-load_test: - needs: detect-changes - if: needs.detect-changes.outputs.load_test == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - token: ${{ secrets.GH_PAT }} - persist-credentials: false - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_PLAN_ROLE_ARN }} - aws-region: ap-northeast-2 - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: ${{ env.TF_VERSION }} - terraform_wrapper: false - - name: Terraform Init - working-directory: environment/load_test - run: terraform init - - name: Terraform Plan - id: plan - working-directory: environment/load_test - run: | - terraform plan -no-color \ - -var-file="../../config/secrets/load_test.tfvars" \ - 2>&1 | tee plan_output.txt - echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - - name: Upload Plan Artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: terraform-plan-load-test - path: environment/load_test/plan_output.txt - - name: Post Plan Comment - if: always() - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const marker = ''; - const planFile = 'environment/load_test/plan_output.txt'; - const output = fs.existsSync(planFile) ? fs.readFileSync(planFile, 'utf8') : ''; - const summary = output.split('\n').find(l => /^Plan:/.test(l)) || output.split('\n').find(l => /No changes/.test(l)) || (output ? '(plan result parse failed)' : 'plan failed before writing output. Check workflow logs.'); - const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const body = `${marker}\n## Terraform Plan: \`load_test\`\n\n${summary}\n\n> Full plan output is kept in the workflow artifact for security. Check [workflow run artifact](${runUrl}).`; - - const { data: comments } = await github.rest.issues.listComments({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - }); - const existing = comments.find(c => c.body.includes(marker)); - if (existing) { - await github.rest.issues.updateComment({ comment_id: existing.id, owner: context.repo.owner, repo: context.repo.repo, body }); - } else { - await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body }); - } - - name: Plan Status Check - if: steps.plan.outputs.exitcode == '1' - run: exit 1 - plan-monitoring: needs: detect-changes if: needs.detect-changes.outputs.monitoring == 'true' @@ -505,7 +438,7 @@ jobs: run: exit 1 trigger-coderabbit: - needs: [plan-bootstrap, plan-global, plan-prod, plan-stage, plan-load_test, plan-monitoring] + needs: [plan-bootstrap, plan-global, plan-prod, plan-stage, plan-monitoring] if: | always() && ( @@ -513,7 +446,6 @@ jobs: needs.plan-global.result == 'success' || needs.plan-global.result == 'failure' || needs.plan-prod.result == 'success' || needs.plan-prod.result == 'failure' || needs.plan-stage.result == 'success' || needs.plan-stage.result == 'failure' || - needs.plan-load_test.result == 'success' || needs.plan-load_test.result == 'failure' || needs.plan-monitoring.result == 'success' || needs.plan-monitoring.result == 'failure' ) runs-on: ubuntu-latest From 4ebeb6ceaca6fbd857f4d8a934116979e6f1913c Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:05:04 +0900 Subject: [PATCH 08/17] =?UTF-8?q?feat:=20k6=20=EC=A0=84=EC=9A=A9=20?= =?UTF-8?q?=EB=B6=80=ED=95=98=20=EC=83=9D=EC=84=B1=20EC2=20=EA=B5=AC?= =?UTF-8?q?=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: load_test Terraform에 k6 전용 EC2와 보안 그룹을 추가했습니다. - 상세내용: stage EC2에는 k6 파일을 배치하지 않도록 app_stack cloud-init 구성을 제거했습니다. - 상세내용: k6 실행에 필요한 기본값은 secret이 아닌 Terraform 기본값과 output으로 관리하도록 정리했습니다. --- environment/load_test/main.tf | 62 ++++++++++++++++++++++++++++++ environment/load_test/output.tf | 25 ++++++++++++ environment/load_test/variables.tf | 36 +++++++++++++++++ environment/stage/main.tf | 2 - modules/app_stack/ec2.tf | 50 ------------------------ modules/app_stack/variables.tf | 12 ------ 6 files changed, 123 insertions(+), 64 deletions(-) diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index 2b5716f..35789d2 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -33,6 +33,21 @@ data "aws_subnets" "target" { } } +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + data "aws_db_instance" "prod" { db_instance_identifier = var.prod_rds_identifier } @@ -85,6 +100,53 @@ resource "aws_security_group_rule" "load_test_db_mysql" { source_security_group_id = each.value } +resource "aws_security_group" "load_generator" { + name = "sc-load-test-generator-sg" + description = "Security group for k6 load generator" + vpc_id = data.aws_subnet.stage_api.vpc_id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "solid-connection-load-test-generator-sg" + } +} + +resource "aws_instance" "load_generator" { + ami = data.aws_ami.ubuntu.id + instance_type = var.load_generator_instance_type + subnet_id = data.aws_instance.stage_api.subnet_id + vpc_security_group_ids = [aws_security_group.load_generator.id] + associate_public_ip_address = true + iam_instance_profile = var.load_generator_instance_profile_name + + root_block_device { + volume_size = var.load_generator_root_volume_size + volume_type = "gp3" + encrypted = true + } + + user_data = <<-EOF + #!/bin/bash + set -eux + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y curl jq + snap install amazon-ssm-agent --classic || true + systemctl enable snap.amazon-ssm-agent.amazon-ssm-agent.service || true + systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true + EOF + + tags = { + Name = "solid-connection-load-test-generator" + } +} + resource "aws_db_subnet_group" "load_test" { name = "sc-load-test-db-subnet-group" subnet_ids = data.aws_subnets.target.ids diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf index ee0bc25..901044a 100644 --- a/environment/load_test/output.tf +++ b/environment/load_test/output.tf @@ -67,3 +67,28 @@ output "prod_db_password_parameter_name" { description = "SSM SecureString parameter name containing the prod DB password" value = var.prod_db_password_parameter_name } + +output "load_generator_instance_id" { + description = "k6 load generator EC2 instance ID" + value = aws_instance.load_generator.id +} + +output "load_generator_private_ip" { + description = "k6 load generator private IP" + value = aws_instance.load_generator.private_ip +} + +output "load_generator_k6_dir" { + description = "Directory where k6 files are placed on the load generator" + value = var.load_generator_k6_dir +} + +output "load_test_target_base_url" { + description = "Default target base URL for k6" + value = var.load_test_target_base_url +} + +output "k6_prometheus_remote_write_url" { + description = "Default Prometheus remote-write URL for k6" + value = var.k6_prometheus_remote_write_url +} diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index efdfa28..9b2e2f6 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -86,3 +86,39 @@ variable "load_test_parameter_prefix" { type = string default = "/solid-connection/loadtest" } + +variable "load_generator_instance_type" { + description = "EC2 instance type for the k6 load generator" + type = string + default = "c7i.xlarge" +} + +variable "load_generator_instance_profile_name" { + description = "Existing IAM instance profile name for the k6 load generator. It must allow SSM RunCommand." + type = string + default = "solid-connection-load-test-generator" +} + +variable "load_generator_root_volume_size" { + description = "Root volume size in GiB for the k6 load generator" + type = number + default = 20 +} + +variable "load_generator_k6_dir" { + description = "Directory where k6 files are placed on the load generator" + type = string + default = "/home/ubuntu/solid-connection-load-test/k6" +} + +variable "load_test_target_base_url" { + description = "Default target base URL for k6" + type = string + default = "https://api.stage.solid-connection.com" +} + +variable "k6_prometheus_remote_write_url" { + description = "Default Prometheus remote-write URL for k6" + type = string + default = "http://132.145.83.182:9090/api/v1/write" +} diff --git a/environment/stage/main.tf b/environment/stage/main.tf index 8e60bb5..a0b0b44 100644 --- a/environment/stage/main.tf +++ b/environment/stage/main.tf @@ -41,6 +41,4 @@ module "stage_stack" { redis_version = var.redis_version redis_exporter_version = var.redis_exporter_version alloy_version = var.alloy_version - - enable_k6_files = true } diff --git a/modules/app_stack/ec2.tf b/modules/app_stack/ec2.tf index e1da678..6734a28 100644 --- a/modules/app_stack/ec2.tf +++ b/modules/app_stack/ec2.tf @@ -22,56 +22,6 @@ data "cloudinit_config" "app_init" { filename = "1_docker_install.sh" } - dynamic "part" { - for_each = var.enable_k6_files ? [1] : [] - - content { - content_type = "text/cloud-config" - content = yamlencode({ - bootcmd = [ - "mkdir -p ${var.k6_install_dir}/script" - ] - write_files = [ - { - path = "${var.k6_install_dir}/createPost.json" - owner = "ubuntu:ubuntu" - permissions = "0644" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/createPost.json") - }, - { - path = "${var.k6_install_dir}/updatePost.json" - owner = "ubuntu:ubuntu" - permissions = "0644" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/updatePost.json") - }, - { - path = "${var.k6_install_dir}/whole-user-flow.js" - owner = "ubuntu:ubuntu" - permissions = "0644" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/whole-user-flow.js") - }, - { - path = "${var.k6_install_dir}/set_up_xk6.sh" - owner = "ubuntu:ubuntu" - permissions = "0755" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/set_up_xk6.sh") - }, - { - path = "${var.k6_install_dir}/script/set-load-test.sh" - owner = "ubuntu:ubuntu" - permissions = "0755" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/script/set-load-test.sh") - } - ] - }) - filename = "2_k6_files.yml" - } - } } # API Server (EC2) diff --git a/modules/app_stack/variables.tf b/modules/app_stack/variables.tf index c1d199c..1c5028d 100644 --- a/modules/app_stack/variables.tf +++ b/modules/app_stack/variables.tf @@ -155,15 +155,3 @@ variable "alloy_version" { description = "Docker image tag for Grafana Alloy" type = string } - -variable "enable_k6_files" { - description = "Whether to place k6 load test files on the API server during cloud-init" - type = bool - default = false -} - -variable "k6_install_dir" { - description = "Directory where k6 load test files are placed" - type = string - default = "/home/ubuntu/solid-connection-load-test/k6" -} From c8a70cc6e5dfec36bc5150a26c9f47780c24cad2 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:05:12 +0900 Subject: [PATCH 09/17] =?UTF-8?q?feat:=20EC2=20=EA=B8=B0=EB=B0=98=20k6=20?= =?UTF-8?q?=EC=8B=A4=ED=96=89=20workflow=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: Load Test Run workflow를 추가해 k6 전용 EC2에서 부하를 생성하도록 했습니다. - 상세내용: loadtest workflow가 전용 AWS_LOAD_TEST_ROLE_ARN 변수를 사용하도록 분리했습니다. - 상세내용: start 스크립트에서 stage k6 동기화를 제거하고 생성된 부하 생성 EC2 정보를 출력하도록 수정했습니다. --- .github/workflows/load-test-run.yml | 84 ++++++++++ .github/workflows/load-test-start.yml | 13 +- .github/workflows/load-test-stop.yml | 2 +- scripts/load_test/run_k6.sh | 215 ++++++++++++++++++++++++++ scripts/load_test/start.sh | 56 +------ 5 files changed, 314 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/load-test-run.yml create mode 100644 scripts/load_test/run_k6.sh diff --git a/.github/workflows/load-test-run.yml b/.github/workflows/load-test-run.yml new file mode 100644 index 0000000..d8675e9 --- /dev/null +++ b/.github/workflows/load-test-run.yml @@ -0,0 +1,84 @@ +name: Load Test Run + +on: + workflow_dispatch: + inputs: + vus: + description: "k6 VUs" + required: true + default: "10" + type: string + iterations: + description: "k6 iterations per VU" + required: true + default: "10" + type: string + max_duration: + description: "k6 max duration" + required: true + default: "15m" + type: string + target_base_url: + description: "Target base URL. Empty uses Terraform default." + required: false + default: "" + type: string + prometheus_remote_write_url: + description: "Prometheus remote-write URL. Empty uses Terraform default." + required: false + default: "" + type: string + +permissions: + id-token: write + contents: read + +concurrency: + group: load-test-environment + cancel-in-progress: false + +env: + TF_VERSION: "1.10.5" + +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }} + aws-region: ap-northeast-2 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Run k6 on load generator + run: | + args=( + --vus "${{ inputs.vus }}" + --iterations "${{ inputs.iterations }}" + --max-duration "${{ inputs.max_duration }}" + ) + + if [ -n "${{ inputs.target_base_url }}" ]; then + args+=(--target-base-url "${{ inputs.target_base_url }}") + fi + + if [ -n "${{ inputs.prometheus_remote_write_url }}" ]; then + args+=(--prometheus-remote-write-url "${{ inputs.prometheus_remote_write_url }}") + fi + + bash scripts/load_test/run_k6.sh "${args[@]}" diff --git a/.github/workflows/load-test-start.yml b/.github/workflows/load-test-start.yml index 80125e4..8755782 100644 --- a/.github/workflows/load-test-start.yml +++ b/.github/workflows/load-test-start.yml @@ -13,6 +13,11 @@ on: required: true default: true type: boolean + load_generator_instance_type: + description: "k6 load generator EC2 instance type" + required: true + default: "c7i.xlarge" + type: string permissions: id-token: write @@ -37,7 +42,7 @@ jobs: - uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }} aws-region: ap-northeast-2 - uses: hashicorp/setup-terraform@v3 @@ -52,6 +57,12 @@ jobs: - name: Start load test environment run: | + export TF_VAR_load_generator_instance_type="${{ inputs.load_generator_instance_type }}" + + if [ -n "${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" ]; then + export TF_VAR_load_generator_instance_profile_name="${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" + fi + args=() if [ "${{ inputs.switch_stage_to_loadtest }}" = "true" ]; then diff --git a/.github/workflows/load-test-stop.yml b/.github/workflows/load-test-stop.yml index 9e5fba7..d5ceffa 100644 --- a/.github/workflows/load-test-stop.yml +++ b/.github/workflows/load-test-stop.yml @@ -37,7 +37,7 @@ jobs: - uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }} aws-region: ap-northeast-2 - uses: hashicorp/setup-terraform@v3 diff --git a/scripts/load_test/run_k6.sh b/scripts/load_test/run_k6.sh new file mode 100644 index 0000000..e2b0477 --- /dev/null +++ b/scripts/load_test/run_k6.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +set -euo pipefail + +TERRAFORM_DIR="environment/load_test" +LOCAL_K6_DIR="config/load-test/k6" +K6_SCRIPT="whole-user-flow.js" +TARGET_BASE_URL="" +PROMETHEUS_REMOTE_WRITE_URL="" +K6_VUS="10" +K6_ITERATIONS="10" +K6_MAX_DURATION="15m" +SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-3600}" + +usage() { + cat <<'EOF' +Usage: scripts/load_test/run_k6.sh [options] + +Options: + --terraform-dir PATH Default: environment/load_test + --local-k6-dir PATH Default: config/load-test/k6 + --script FILE Default: whole-user-flow.js + --target-base-url URL Default: Terraform output load_test_target_base_url + --prometheus-remote-write-url URL Default: Terraform output k6_prometheus_remote_write_url + --vus VALUE Default: 10 + --iterations VALUE Default: 10 + --max-duration VALUE Default: 15m + --ssm-command-timeout-seconds Default: 3600 + -h, --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --local-k6-dir) LOCAL_K6_DIR="$2"; shift 2 ;; + --script) K6_SCRIPT="$2"; shift 2 ;; + --target-base-url) TARGET_BASE_URL="$2"; shift 2 ;; + --prometheus-remote-write-url) PROMETHEUS_REMOTE_WRITE_URL="$2"; shift 2 ;; + --vus) K6_VUS="$2"; shift 2 ;; + --iterations) K6_ITERATIONS="$2"; shift 2 ;; + --max-duration) K6_MAX_DURATION="$2"; shift 2 ;; + --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Required command not found: $1" >&2 + exit 1 + fi +} + +require_command terraform +require_command aws +require_command jq +require_command base64 + +tf_output() { + terraform -chdir="$TERRAFORM_DIR" output -raw "$1" +} + +send_ssm_command() { + local instance_id="$1" + local comment="$2" + local commands_json="$3" + + local command_id + local started_at + command_id="$(aws ssm send-command \ + --instance-ids "$instance_id" \ + --document-name "AWS-RunShellScript" \ + --comment "$comment" \ + --parameters "$commands_json" \ + --query "Command.CommandId" \ + --output text)" + started_at="$(date +%s)" + + local status + while true; do + sleep 5 + status="$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "Status" \ + --output text 2>/dev/null || true)" + + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command timed out after ${SSM_COMMAND_TIMEOUT_SECONDS}s: $comment" >&2 + exit 1 + fi + + case "$status" in + Pending|InProgress|Delayed|"") continue ;; + Success) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "StandardOutputContent" \ + --output text || true + break + ;; + *) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command failed with status $status: $comment" >&2 + exit 1 + ;; + esac + done +} + +wait_for_ssm() { + local instance_id="$1" + local started_at + started_at="$(date +%s)" + + while true; do + local ping_status + ping_status="$(aws ssm describe-instance-information \ + --filters "Key=InstanceIds,Values=${instance_id}" \ + --query "InstanceInformationList[0].PingStatus" \ + --output text 2>/dev/null || true)" + + if [[ "$ping_status" == "Online" ]]; then + break + fi + + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + echo "SSM agent did not become online after ${SSM_COMMAND_TIMEOUT_SECONDS}s: ${instance_id}" >&2 + exit 1 + fi + + sleep 10 + done +} + +file_base64() { + base64 "$1" | tr -d '\n' +} + +sync_file() { + local instance_id="$1" + local target_dir="$2" + local relative_path="$3" + local source_path="${LOCAL_K6_DIR}/${relative_path}" + + if [[ ! -f "$source_path" ]]; then + echo "Missing k6 file: $source_path" >&2 + exit 1 + fi + + local commands_json + commands_json="$(jq -cn \ + --arg target "${target_dir}/${relative_path}" \ + --arg content "$(file_base64 "$source_path")" \ + '{ + commands: [ + "set -euo pipefail", + "mkdir -p \"$(dirname \"\($target)\")\"", + "printf %s \($content | @sh) | base64 -d > \($target | @sh)" + ] + }')" + + send_ssm_command "$instance_id" "Sync ${relative_path} to load generator" "$commands_json" +} + +terraform -chdir="$TERRAFORM_DIR" init + +load_generator_instance_id="$(tf_output load_generator_instance_id)" +load_generator_k6_dir="$(tf_output load_generator_k6_dir)" +tf_target_base_url="$(tf_output load_test_target_base_url)" +tf_prometheus_remote_write_url="$(tf_output k6_prometheus_remote_write_url)" + +TARGET_BASE_URL="${TARGET_BASE_URL:-$tf_target_base_url}" +PROMETHEUS_REMOTE_WRITE_URL="${PROMETHEUS_REMOTE_WRITE_URL:-$tf_prometheus_remote_write_url}" + +wait_for_ssm "$load_generator_instance_id" + +for relative_path in \ + "createPost.json" \ + "updatePost.json" \ + "whole-user-flow.js" \ + "set_up_xk6.sh" \ + "script/set-load-test.sh"; do + sync_file "$load_generator_instance_id" "$load_generator_k6_dir" "$relative_path" +done + +run_commands_json="$(jq -cn \ + --arg k6_dir "$load_generator_k6_dir" \ + --arg script "$K6_SCRIPT" \ + --arg target_base_url "$TARGET_BASE_URL" \ + --arg prometheus_url "$PROMETHEUS_REMOTE_WRITE_URL" \ + --arg vus "$K6_VUS" \ + --arg iterations "$K6_ITERATIONS" \ + --arg max_duration "$K6_MAX_DURATION" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($k6_dir)", + "chmod +x set_up_xk6.sh script/set-load-test.sh", + "chown -R ubuntu:ubuntu \($k6_dir)", + "if [ ! -x ./k6 ]; then sudo -u ubuntu -H ./set_up_xk6.sh; fi", + "sudo -u ubuntu -H env BASE_URL=\($target_base_url | @sh) K6_PROMETHEUS_RW_SERVER_URL=\($prometheus_url | @sh) K6_VUS=\($vus | @sh) K6_ITERATIONS=\($iterations | @sh) K6_MAX_DURATION=\($max_duration | @sh) ./k6 run \($script | @sh)" + ] + }')" + +send_ssm_command "$load_generator_instance_id" "Run k6 load test" "$run_commands_json" diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh index 21ab43d..e3a7414 100644 --- a/scripts/load_test/start.sh +++ b/scripts/load_test/start.sh @@ -12,8 +12,6 @@ LOADTEST_DB_PASSWORD_PARAMETER="" SWITCH_STAGE_TO_LOADTEST="false" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" -STAGE_K6_DIR="/home/ubuntu/solid-connection-load-test/k6" -LOCAL_K6_DIR="config/load-test/k6" SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-1800}" SKIP_TERRAFORM_APPLY="false" SKIP_DATA_COPY="false" @@ -34,8 +32,6 @@ Options: --switch-stage-to-loadtest Restart stage app through SSM with dev,loadtest profiles --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml - --stage-k6-dir PATH Default: /home/ubuntu/solid-connection-load-test/k6 - --local-k6-dir PATH Default: config/load-test/k6 --ssm-command-timeout-seconds Default: 1800 --skip-terraform-apply --skip-data-copy @@ -56,8 +52,6 @@ while [[ $# -gt 0 ]]; do --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; - --stage-k6-dir) STAGE_K6_DIR="$2"; shift 2 ;; - --local-k6-dir) LOCAL_K6_DIR="$2"; shift 2 ;; --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; --skip-data-copy) SKIP_DATA_COPY="true"; shift ;; @@ -85,7 +79,6 @@ require_command() { require_command terraform require_command aws require_command jq -require_command base64 tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" @@ -140,51 +133,6 @@ send_ssm_command() { done } -file_base64() { - base64 "$1" | tr -d '\n' -} - -sync_stage_k6_files() { - local instance_id="$1" - local commands - commands="$(jq -cn \ - --arg target_dir "$STAGE_K6_DIR" \ - '{ commands: ["set -euo pipefail", "mkdir -p \($target_dir)/script"] }')" - - local relative_path - for relative_path in \ - "createPost.json" \ - "updatePost.json" \ - "whole-user-flow.js" \ - "set_up_xk6.sh" \ - "script/set-load-test.sh"; do - local source_path="${LOCAL_K6_DIR}/${relative_path}" - if [[ ! -f "$source_path" ]]; then - echo "Missing k6 file: $source_path" >&2 - exit 1 - fi - - commands="$(jq -cn \ - --argjson current "$commands" \ - --arg target "${STAGE_K6_DIR}/${relative_path}" \ - --arg content "$(file_base64 "$source_path")" \ - '$current | .commands += [ - "mkdir -p \"$(dirname \"\($target)\")\"", - "printf %s \($content | @sh) | base64 -d > \($target | @sh)" - ]')" - done - - commands="$(jq -cn \ - --argjson current "$commands" \ - --arg target_dir "$STAGE_K6_DIR" \ - '$current | .commands += [ - "chmod +x \($target_dir)/set_up_xk6.sh \($target_dir)/script/set-load-test.sh", - "chown -R ubuntu:ubuntu \($target_dir)" - ]')" - - send_ssm_command "$instance_id" "Sync k6 files to stage EC2" "$commands" -} - delete_temp_parameters() { aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" >/dev/null 2>&1 || true aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" >/dev/null 2>&1 || true @@ -204,6 +152,7 @@ prod_endpoint="$(tf_output prod_rds_endpoint)" prod_port="$(tf_output prod_rds_port)" loadtest_endpoint="$(tf_output load_test_rds_endpoint)" loadtest_port="$(tf_output load_test_rds_port)" +load_generator_instance_id="$(tf_output load_generator_instance_id)" loadtest_db_name="$(tf_output load_test_db_name)" tf_prod_db_username_parameter="$(tf_output prod_db_username_parameter_name)" tf_prod_db_password_parameter="$(tf_output prod_db_password_parameter_name)" @@ -294,8 +243,6 @@ if [[ "$SKIP_DATA_COPY" != "true" ]]; then fi if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then - sync_stage_k6_files "$stage_instance_id" - stage_commands_json="$(jq -cn \ --arg app_dir "$STAGE_APP_DIR" \ --arg compose_file "$STAGE_COMPOSE_FILE" \ @@ -318,5 +265,6 @@ fi echo "Load test environment is ready." echo "RDS endpoint: ${loadtest_endpoint}:${loadtest_port}" +echo "Load generator instance: ${load_generator_instance_id}" echo "Stage instance: ${stage_instance_id}" echo "Stage public IP: ${stage_public_ip}" From b097ca95d25e339fe4dc73bc5779f3359a96daff Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:05:22 +0900 Subject: [PATCH 10/17] =?UTF-8?q?fix:=20k6=20=EC=8B=A4=ED=96=89=20?= =?UTF-8?q?=EC=84=A4=EC=A0=95=EA=B3=BC=20=EC=9D=91=EB=8B=B5=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: Prometheus remote-write 설정을 환경 변수 기반으로 일관되게 export하도록 수정했습니다. - 상세내용: k6 VU, iteration, duration, target URL을 실행 시 주입할 수 있도록 변경했습니다. - 상세내용: 대학, 어학 점수, GPA 응답이 비어 있을 때 명확히 fail하도록 검증을 추가했습니다. --- config/load-test/k6/set_up_xk6.sh | 4 ++-- config/load-test/k6/whole-user-flow.js | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/load-test/k6/set_up_xk6.sh b/config/load-test/k6/set_up_xk6.sh index 01ad6d6..eb1f254 100644 --- a/config/load-test/k6/set_up_xk6.sh +++ b/config/load-test/k6/set_up_xk6.sh @@ -11,8 +11,8 @@ export GOPATH=${BASE_DIR}/go-workspace export PATH=$PATH:$GOROOT/bin:$GOPATH/bin export XK6_BIN=${GOPATH}/bin/xk6 export K6_OUT=xk6-prometheus-rw -export K6_PROMETHEUS_RW_SERVER_URL=http://132.145.83.182:9090/api/v1/write -export K6_PROMETHEUS_RW_TREND_STATS="p(90),p(95),p(99),avg,min,max" +export K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL:-http://132.145.83.182:9090/api/v1/write} +export K6_PROMETHEUS_RW_TREND_STATS="${K6_PROMETHEUS_RW_TREND_STATS:-p(90),p(95),p(99),avg,min,max}" { echo "export BASE_DIR=${BASE_DIR}" diff --git a/config/load-test/k6/whole-user-flow.js b/config/load-test/k6/whole-user-flow.js index a40f82c..8195806 100644 --- a/config/load-test/k6/whole-user-flow.js +++ b/config/load-test/k6/whole-user-flow.js @@ -13,16 +13,16 @@ const time = (() => { return `${mm}/${dd} ${hh}:${min}`; })(); -const BASE_URL = 'https://api.stage.solid-connection.com'; +const BASE_URL = __ENV.BASE_URL || 'https://api.stage.solid-connection.com'; const testId = 'whole-user-flow'; export const options = { scenarios: { user_flow: { executor: 'per-vu-iterations', // VU별 반복 - vus: 10, // VU - iterations: 10, // VU 한 명당 실행할 횟수 - maxDuration: '15m', // 여유로 잡아 두기 + vus: Number(__ENV.K6_VUS || 10), + iterations: Number(__ENV.K6_ITERATIONS || 10), + maxDuration: __ENV.K6_MAX_DURATION || '15m', }, }, tags: { From 4ad54c46193cd550e2578a90067172e491e12c22 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:05:28 +0900 Subject: [PATCH 11/17] =?UTF-8?q?docs:=20EC2=20=EA=B8=B0=EB=B0=98=20?= =?UTF-8?q?=EB=B6=80=ED=95=98=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=A0=88?= =?UTF-8?q?=EC=B0=A8=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: Start, Run, Stop workflow 기준의 부하 테스트 실행 흐름을 문서화했습니다. - 상세내용: secret에 새로 추가할 값이 없고 민감하지 않은 값은 workflow 입력과 기본값으로 관리한다는 점을 명시했습니다. - 상세내용: stage EC2가 아닌 k6 전용 EC2에서 부하를 생성하는 구조를 설명했습니다. --- scripts/load_test/README.md | 61 ++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 3647c44..dacc960 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -1,13 +1,32 @@ # 부하 테스트 자동화 -부하 테스트용 임시 RDS를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작과 종료는 GitHub Actions에서 수동으로 실행합니다. +부하 테스트용 임시 RDS와 k6 전용 EC2를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작, 실행, 종료는 GitHub Actions에서 수동으로 실행합니다. + +## secret에 필요한 값 + +EC2 기반 k6 실행을 위해 새로 `config/secrets/load_test.tfvars`에 추가해야 하는 값은 없습니다. + +기존 loadtest Terraform에 필요한 민감값만 secret에 둡니다. + +- `load_test_db_username_parameter_name` +- `load_test_db_password_parameter_name` + +아래 값들은 민감값이 아니므로 secret에 넣지 않고 Terraform 기본값이나 GitHub Actions 입력값으로 처리합니다. + +- k6 전용 EC2 instance type: 기본값 `c7i.xlarge` +- k6 전용 EC2 IAM instance profile 이름: 기본값 `solid-connection-load-test-generator` +- k6 target URL: 기본값 `https://api.stage.solid-connection.com` +- k6 VU, iterations, max duration: **Load Test Run** workflow 입력값 +- Prometheus remote-write URL: 기본값 사용 또는 **Load Test Run** workflow 입력값 + +기본 instance profile 이름을 쓰지 않을 경우 secret이 아니라 GitHub Actions variable 또는 `TF_VAR_load_generator_instance_profile_name`으로 덮어씁니다. ## 원칙 - 사람이 로컬에서 `terraform apply` 또는 `terraform destroy`를 직접 실행하지 않습니다. -- 시작은 **Actions > Load Test Start** workflow로 실행합니다. -- 종료는 **Actions > Load Test Stop** workflow로 실행합니다. -- SSH private key는 사용하지 않습니다. stage/prod EC2 작업은 SSM RunCommand로 실행합니다. +- stage EC2는 부하를 받는 대상이므로 stage EC2에서 k6를 실행하지 않습니다. +- k6는 loadtest Terraform이 생성한 별도 EC2에서 실행합니다. +- SSH private key는 사용하지 않습니다. stage/prod/load-generator EC2 작업은 SSM RunCommand로 실행합니다. ## 시작 @@ -25,13 +44,27 @@ workflow는 GitHub Actions runner에서 `environment/load_test`의 Terraform을 ## 시작 시 수행 작업 1. GitHub Actions가 `environment/load_test`에서 Terraform apply를 실행합니다. -2. loadtest RDS와 보안 그룹을 생성합니다. RDS는 stage EC2가 속한 VPC/subnet 기준으로 생성됩니다. +2. loadtest RDS, 보안 그룹, k6 전용 EC2를 생성합니다. 3. loadtest datasource 값을 Parameter Store에 작성합니다. 4. prod DB와 loadtest DB 접속 정보를 Parameter Store에서 읽습니다. 5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS로 복원합니다. -6. stage EC2에 k6 파일을 `/home/ubuntu/solid-connection-load-test/k6` 경로로 동기화합니다. -7. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. -8. 데이터 이관용 임시 Parameter Store 값을 삭제합니다. +6. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +7. 데이터 이관용 임시 Parameter Store 값을 삭제합니다. + +## 실행 + +1. GitHub에서 **Actions > Load Test Run**을 엽니다. +2. **Run workflow**를 클릭합니다. +3. VU, iterations, max duration을 입력하고 실행합니다. + +workflow는 `scripts/load_test/run_k6.sh`를 실행합니다. + +실행 시 수행 작업: + +1. Terraform output에서 k6 전용 EC2 ID와 기본 실행값을 읽습니다. +2. SSM RunCommand로 k6 파일을 k6 전용 EC2에 배치합니다. +3. k6 전용 EC2에서 `set_up_xk6.sh`로 k6 binary를 준비합니다. +4. k6 전용 EC2에서 `whole-user-flow.js`를 실행합니다. ## 종료 @@ -44,12 +77,10 @@ workflow는 GitHub Actions runner에서 `environment/load_test`의 Terraform을 - `restore_stage_dev`: stage 앱을 기존 dev compose 구성으로 되돌립니다. - `destroy_rds`: loadtest Terraform stack을 제거합니다. -workflow는 `scripts/load_test/stop.sh`를 실행하고, 선택값에 따라 stage 복구와 Terraform destroy를 수행합니다. +workflow는 `scripts/load_test/stop.sh`를 실행하고, 선택값에 따라 stage 복구와 Terraform destroy를 수행합니다. Terraform destroy에는 loadtest RDS와 k6 전용 EC2 제거가 포함됩니다. ## k6 파일 -stage EC2를 새로 생성하는 경우 Terraform cloud-init이 k6 파일을 배치합니다. 기존 stage EC2는 cloud-init이 다시 실행되지 않으므로, **Load Test Start** workflow가 실행될 때 SSM으로 k6 파일을 다시 동기화합니다. - 포함 파일: - `createPost.json` @@ -58,13 +89,7 @@ stage EC2를 새로 생성하는 경우 Terraform cloud-init이 k6 파일을 배 - `set_up_xk6.sh` - `script/set-load-test.sh` -stage EC2에 접속해 수동으로 실행해야 한다면 다음 경로에서 실행합니다. - -```bash -cd /home/ubuntu/solid-connection-load-test/k6 -./set_up_xk6.sh -./script/set-load-test.sh -``` +이 파일들은 stage EC2가 아니라 k6 전용 EC2에 동기화됩니다. ## 참고 From 628043af4cc74f64ace023a0cc80bc5ab530c487 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Fri, 29 May 2026 01:51:10 +0900 Subject: [PATCH 12/17] =?UTF-8?q?feat:=20k6=20=EC=9D=91=EB=8B=B5=20?= =?UTF-8?q?=EA=B2=80=EC=A6=9D=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - HTTP 실패와 JSON 파싱 실패를 명시적으로 처리하도록 수정 - 빈 배열과 누락된 id를 역참조 전에 검증하도록 추가 --- config/load-test/k6/whole-user-flow.js | 33 ++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/config/load-test/k6/whole-user-flow.js b/config/load-test/k6/whole-user-flow.js index 8195806..b0a9166 100644 --- a/config/load-test/k6/whole-user-flow.js +++ b/config/load-test/k6/whole-user-flow.js @@ -293,9 +293,21 @@ function getGPAs(auth) { }); } -function requireArray(value, name) { +function requireJson(res, name) { + if (res.status < 200 || res.status >= 300) { + fail(`${name} failed with status ${res.status}: ${res.body}`); + } + + try { + return res.json(); + } catch (error) { + fail(`${name} returned invalid JSON: ${error.message}`); + } +} + +function requireArray(value, name, status) { if (!Array.isArray(value) || value.length === 0) { - fail(`${name} response is empty or invalid`); + fail(`${name} response is empty or invalid (status: ${status})`); } return value; } @@ -345,7 +357,8 @@ export default function () { getRecommendedUniversities(auth); const uniSearchRes = searchUniversities(''); // 이번학기 열린 대학 중 랜덤하게 id 가져오기 - const uniList = requireArray(uniSearchRes.json(), 'universities/search'); + const uniSearchBody = requireJson(uniSearchRes, 'searchUniversities'); + const uniList = requireArray(uniSearchBody, 'searchUniversities', uniSearchRes.status); const universityId = requireId(uniList[Math.floor(Math.random() * uniList.length)], 'universities/search item'); likeUniversity(universityId, auth); @@ -372,11 +385,21 @@ export default function () { deletePost(postId, auth); const langRes = getLanguageTests(auth); - const langList = requireArray(langRes.json().languageTestScoreStatusResponseList, 'scores/language-tests'); + const langBody = requireJson(langRes, 'getLanguageTests'); + const langList = requireArray( + langBody && langBody.languageTestScoreStatusResponseList, + 'getLanguageTests.languageTestScoreStatusResponseList', + langRes.status + ); const languageTestScoreId = requireId(langList[0], 'scores/language-tests item'); const gpaRes = getGPAs(auth); - const gpaList = requireArray(gpaRes.json().gpaScoreStatusResponseList, 'scores/gpas'); + const gpaBody = requireJson(gpaRes, 'getGPAs'); + const gpaList = requireArray( + gpaBody && gpaBody.gpaScoreStatusResponseList, + 'getGPAs.gpaScoreStatusResponseList', + gpaRes.status + ); const gpaScoreId = requireId(gpaList[0], 'scores/gpas item'); apply(gpaScoreId, languageTestScoreId, universityId, auth); From 6f2247a41280130f1589932a9aefbf9d4f34d4e6 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Fri, 29 May 2026 01:51:16 +0900 Subject: [PATCH 13/17] =?UTF-8?q?feat:=20RDS=20=EC=8A=A4=EB=83=85=EC=83=B7?= =?UTF-8?q?=20=EA=B8=B0=EB=B0=98=20=EB=B6=80=ED=95=98=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=8A=B8=20=EA=B5=AC=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - prod RDS 최신 자동 스냅샷에서 load-test RDS를 복원하도록 수정 - 덤프 복제 입력과 스크립트 로직을 제거하도록 수정 - load generator 타입을 c7i.large로 고정하도록 수정 - 현재 Start/Run/Stop 흐름에 맞춰 README를 갱신하도록 수정 --- .github/workflows/load-test-start.yml | 16 +-- environment/load_test/main.tf | 28 +++--- environment/load_test/output.tf | 2 +- environment/load_test/variables.tf | 14 ++- scripts/load_test/README.md | 140 +++++++++++++++----------- scripts/load_test/start.sh | 113 --------------------- 6 files changed, 104 insertions(+), 209 deletions(-) diff --git a/.github/workflows/load-test-start.yml b/.github/workflows/load-test-start.yml index 8755782..c5e02b6 100644 --- a/.github/workflows/load-test-start.yml +++ b/.github/workflows/load-test-start.yml @@ -8,16 +8,6 @@ on: required: true default: true type: boolean - copy_prod_data: - description: "Copy prod RDS data to load test RDS" - required: true - default: true - type: boolean - load_generator_instance_type: - description: "k6 load generator EC2 instance type" - required: true - default: "c7i.xlarge" - type: string permissions: id-token: write @@ -57,7 +47,7 @@ jobs: - name: Start load test environment run: | - export TF_VAR_load_generator_instance_type="${{ inputs.load_generator_instance_type }}" + export TF_VAR_load_generator_instance_type="c7i.large" if [ -n "${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" ]; then export TF_VAR_load_generator_instance_profile_name="${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" @@ -69,8 +59,4 @@ jobs: args+=(--switch-stage-to-loadtest) fi - if [ "${{ inputs.copy_prod_data }}" != "true" ]; then - args+=(--skip-data-copy) - fi - bash scripts/load_test/start.sh "${args[@]}" diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index 35789d2..f68c207 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -52,19 +52,22 @@ data "aws_db_instance" "prod" { db_instance_identifier = var.prod_rds_identifier } -data "aws_ssm_parameter" "db_root_username" { - name = var.load_test_db_username_parameter_name +data "aws_db_snapshot" "latest_prod" { + db_instance_identifier = var.prod_rds_identifier + most_recent = true + snapshot_type = "automated" +} + +data "aws_ssm_parameter" "prod_db_username" { + name = var.prod_db_username_parameter_name } -data "aws_ssm_parameter" "db_root_password" { - name = var.load_test_db_password_parameter_name +data "aws_ssm_parameter" "prod_db_password" { + name = var.prod_db_password_parameter_name with_decryption = true } locals { - db_root_username = data.aws_ssm_parameter.db_root_username.value - db_root_password = data.aws_ssm_parameter.db_root_password.value - source_security_group_ids = setunion( data.aws_instance.prod_api.vpc_security_group_ids, data.aws_instance.stage_api.vpc_security_group_ids @@ -158,14 +161,9 @@ resource "aws_db_subnet_group" "load_test" { resource "aws_db_instance" "load_test" { identifier = var.rds_identifier - allocated_storage = var.allocated_storage - engine = "mysql" - engine_version = var.db_engine_version instance_class = var.db_instance_class - db_name = var.db_name - username = local.db_root_username - password = local.db_root_password parameter_group_name = var.db_parameter_group_name + snapshot_identifier = data.aws_db_snapshot.latest_prod.id db_subnet_group_name = aws_db_subnet_group.load_test.name vpc_security_group_ids = [aws_security_group.load_test_db.id] publicly_accessible = false @@ -192,14 +190,14 @@ resource "aws_ssm_parameter" "load_test_datasource_url" { resource "aws_ssm_parameter" "load_test_datasource_username" { name = "${var.load_test_parameter_prefix}/spring.datasource.username" type = "String" - value = local.db_root_username + value = data.aws_ssm_parameter.prod_db_username.value overwrite = true } resource "aws_ssm_parameter" "load_test_datasource_password" { name = "${var.load_test_parameter_prefix}/spring.datasource.password" type = "SecureString" - value = local.db_root_password + value = data.aws_ssm_parameter.prod_db_password.value key_id = var.ssm_kms_key_id overwrite = true tier = "Standard" diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf index 901044a..2299413 100644 --- a/environment/load_test/output.tf +++ b/environment/load_test/output.tf @@ -29,7 +29,7 @@ output "prod_rds_port" { } output "prod_api_instance_id" { - description = "Prod API EC2 instance ID used to run migration commands" + description = "Prod API EC2 instance ID whose security group can access load-test RDS" value = data.aws_instance.prod_api.id } diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index 9b2e2f6..d0dc556 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -15,8 +15,10 @@ variable "allocated_storage" { } variable "db_engine_version" { - description = "MySQL engine version" + description = "Deprecated. The load-test RDS is restored from the latest prod snapshot." type = string + default = null + nullable = true } variable "db_parameter_group_name" { @@ -31,13 +33,17 @@ variable "db_name" { } variable "load_test_db_username_parameter_name" { - description = "SSM parameter name containing the load test DB root username" + description = "Deprecated. Load-test datasource credentials are copied from prod datasource parameters." type = string + default = null + nullable = true } variable "load_test_db_password_parameter_name" { - description = "SSM SecureString parameter name containing the load test DB root password" + description = "Deprecated. Load-test datasource credentials are copied from prod datasource parameters." type = string + default = null + nullable = true } variable "prod_db_username_parameter_name" { @@ -70,7 +76,7 @@ variable "prod_rds_identifier" { } variable "prod_api_instance_name" { - description = "Name tag of the prod API EC2 instance used to run dump/restore" + description = "Name tag of the prod API EC2 instance whose security group can access load-test RDS" type = string default = "solid-connection-server-prod" } diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index dacc960..7b81f7f 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -1,87 +1,79 @@ # 부하 테스트 자동화 -부하 테스트용 임시 RDS와 k6 전용 EC2를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작, 실행, 종료는 GitHub Actions에서 수동으로 실행합니다. +이 디렉터리는 부하 테스트용 GitHub Actions workflow에서 사용하는 스크립트를 담고 있습니다. -## secret에 필요한 값 +전체 흐름은 다음과 같습니다. -EC2 기반 k6 실행을 위해 새로 `config/secrets/load_test.tfvars`에 추가해야 하는 값은 없습니다. +1. **Load Test Start**: 임시 부하 테스트 인프라를 만들고 stage를 준비합니다. +2. **Load Test Run**: 별도 k6 부하 생성 EC2에서 k6를 실행합니다. +3. **Load Test Stop**: stage를 복구하고 임시 부하 테스트 스택을 제거합니다. -기존 loadtest Terraform에 필요한 민감값만 secret에 둡니다. +## 규칙 -- `load_test_db_username_parameter_name` -- `load_test_db_password_parameter_name` +- 환경 Terraform에 대해 로컬에서 `terraform apply` 또는 `terraform destroy`를 실행하지 않습니다. +- 시작, 실행, 종료는 GitHub Actions에서 수행합니다. +- k6는 stage EC2에서 실행하지 않습니다. 부하 테스트용 Terraform이 생성한 별도 load-generator EC2에서 실행합니다. +- SSH private key를 사용하지 않습니다. EC2 명령은 SSM RunCommand로 실행합니다. -아래 값들은 민감값이 아니므로 secret에 넣지 않고 Terraform 기본값이나 GitHub Actions 입력값으로 처리합니다. +## 필요한 설정 -- k6 전용 EC2 instance type: 기본값 `c7i.xlarge` -- k6 전용 EC2 IAM instance profile 이름: 기본값 `solid-connection-load-test-generator` -- k6 target URL: 기본값 `https://api.stage.solid-connection.com` -- k6 VU, iterations, max duration: **Load Test Run** workflow 입력값 -- Prometheus remote-write URL: 기본값 사용 또는 **Load Test Run** workflow 입력값 +`environment/load_test`는 `config/secrets/load_test.tfvars`를 사용합니다. -기본 instance profile 이름을 쓰지 않을 경우 secret이 아니라 GitHub Actions variable 또는 `TF_VAR_load_generator_instance_profile_name`으로 덮어씁니다. +스냅샷 복원 방식에서는 load-test DB root 계정을 별도로 만들지 않습니다. load-test datasource username/password는 prod datasource Parameter Store 값을 복사해 사용합니다. -## 원칙 +주요 확인값: -- 사람이 로컬에서 `terraform apply` 또는 `terraform destroy`를 직접 실행하지 않습니다. -- stage EC2는 부하를 받는 대상이므로 stage EC2에서 k6를 실행하지 않습니다. -- k6는 loadtest Terraform이 생성한 별도 EC2에서 실행합니다. -- SSH private key는 사용하지 않습니다. stage/prod/load-generator EC2 작업은 SSM RunCommand로 실행합니다. +- `prod_rds_identifier`: snapshot을 조회할 prod RDS identifier +- `kms_key_arn`: 복원된 load-test RDS storage encryption에 사용할 KMS key ARN +- `prod_db_username_parameter_name`: 기본값 `/solid-connection/prod/spring.datasource.username` +- `prod_db_password_parameter_name`: 기본값 `/solid-connection/prod/spring.datasource.password` -## 시작 +그 외 부하 테스트 설정값은 Terraform 기본값, GitHub Actions variable, workflow 입력값으로 처리합니다. -1. GitHub에서 **Actions > Load Test Start**를 엽니다. -2. **Run workflow**를 클릭합니다. -3. 필요한 입력값을 선택하고 실행합니다. +## Load Test Start -입력값: +GitHub에서 **Actions > Load Test Start**를 수동 실행합니다. -- `switch_stage_to_loadtest`: stage 앱을 `dev,loadtest` 프로필로 재기동합니다. -- `copy_prod_data`: prod RDS 데이터를 loadtest RDS로 복사합니다. +입력값: -workflow는 GitHub Actions runner에서 `environment/load_test`의 Terraform을 apply하고 `scripts/load_test/start.sh`를 실행합니다. +- `switch_stage_to_loadtest`: `true` 또는 `false` + - `true`이면 데이터 준비 후 stage 앱을 `dev,loadtest` profile로 재기동합니다. -## 시작 시 수행 작업 +Start workflow 동작: 1. GitHub Actions가 `environment/load_test`에서 Terraform apply를 실행합니다. -2. loadtest RDS, 보안 그룹, k6 전용 EC2를 생성합니다. -3. loadtest datasource 값을 Parameter Store에 작성합니다. -4. prod DB와 loadtest DB 접속 정보를 Parameter Store에서 읽습니다. -5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS로 복원합니다. -6. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. -7. 데이터 이관용 임시 Parameter Store 값을 삭제합니다. - -## 실행 - -1. GitHub에서 **Actions > Load Test Run**을 엽니다. -2. **Run workflow**를 클릭합니다. -3. VU, iterations, max duration을 입력하고 실행합니다. - -workflow는 `scripts/load_test/run_k6.sh`를 실행합니다. +2. Terraform이 최신 prod RDS 자동 snapshot을 조회합니다. +3. Terraform이 해당 snapshot에서 load-test RDS를 복원합니다. +4. Terraform이 보안 그룹과 `c7i.large` 타입의 k6 load-generator EC2를 생성합니다. +5. Terraform이 load-test datasource 값을 Parameter Store에 기록합니다. + - datasource URL은 복원된 load-test RDS endpoint를 사용합니다. + - datasource username/password는 prod datasource Parameter Store 값을 사용합니다. +6. `scripts/load_test/start.sh`가 Terraform output에서 필요한 값을 읽습니다. +7. `switch_stage_to_loadtest=true`이면 stage 앱을 `dev,loadtest` profile로 재기동합니다. -실행 시 수행 작업: +## Load Test Run -1. Terraform output에서 k6 전용 EC2 ID와 기본 실행값을 읽습니다. -2. SSM RunCommand로 k6 파일을 k6 전용 EC2에 배치합니다. -3. k6 전용 EC2에서 `set_up_xk6.sh`로 k6 binary를 준비합니다. -4. k6 전용 EC2에서 `whole-user-flow.js`를 실행합니다. - -## 종료 - -1. GitHub에서 **Actions > Load Test Stop**을 엽니다. -2. **Run workflow**를 클릭합니다. -3. 필요한 입력값을 선택하고 실행합니다. +GitHub에서 **Actions > Load Test Run**을 수동 실행합니다. 입력값: -- `restore_stage_dev`: stage 앱을 기존 dev compose 구성으로 되돌립니다. -- `destroy_rds`: loadtest Terraform stack을 제거합니다. +- `vus`: k6 virtual user 수입니다. 예: `10` +- `iterations`: VU당 반복 횟수입니다. 예: `10` +- `max_duration`: 최대 실행 시간입니다. 예: `30s`, `5m`, `15m`, `1h` +- `target_base_url` + - 선택값입니다. 비워두면 Terraform output `load_test_target_base_url`을 사용합니다. +- `prometheus_remote_write_url` + - 선택값입니다. 비워두면 Terraform output `k6_prometheus_remote_write_url`을 사용합니다. -workflow는 `scripts/load_test/stop.sh`를 실행하고, 선택값에 따라 stage 복구와 Terraform destroy를 수행합니다. Terraform destroy에는 loadtest RDS와 k6 전용 EC2 제거가 포함됩니다. +Run workflow 동작: -## k6 파일 +1. `scripts/load_test/run_k6.sh`가 Terraform output에서 load-generator EC2 ID와 k6 기본값을 읽습니다. +2. load-generator EC2의 SSM agent가 online 상태가 될 때까지 기다립니다. +3. SSM RunCommand로 k6 파일을 load-generator EC2에 동기화합니다. +4. k6 binary가 없으면 `set_up_xk6.sh`로 Prometheus remote-write 지원이 포함된 k6를 빌드합니다. +5. load-generator EC2에서 `whole-user-flow.js`를 실행합니다. -포함 파일: +동기화되는 k6 파일: - `createPost.json` - `updatePost.json` @@ -89,12 +81,38 @@ workflow는 `scripts/load_test/stop.sh`를 실행하고, 선택값에 따라 sta - `set_up_xk6.sh` - `script/set-load-test.sh` -이 파일들은 stage EC2가 아니라 k6 전용 EC2에 동기화됩니다. +## 결과 확인 + +간단한 실행 결과는 **Load Test Run** GitHub Actions 로그에서 확인합니다. + +k6 스크립트는 Prometheus remote-write로도 지표를 전송합니다. + +- 기본 remote-write URL은 Terraform output `k6_prometheus_remote_write_url`을 사용합니다. +- workflow 입력값 `prometheus_remote_write_url`로 override할 수 있습니다. +- k6 지표에는 요청 수, 실패율, 응답 시간, p90, p95, p99, 평균, 최소, 최대값이 포함됩니다. +- API 호출에는 `name`, `testid`, `time` tag가 붙어 endpoint와 실행 시점별로 필터링할 수 있습니다. + +## Load Test Stop + +GitHub에서 **Actions > Load Test Stop**을 수동 실행합니다. + +입력값: + +- `restore_stage_dev`: `true` 또는 `false` + - `true`이면 stage 앱을 기존 dev compose 구성으로 되돌립니다. +- `destroy_rds`: `true` 또는 `false` + - `true`이면 load-test Terraform stack을 destroy합니다. + +Stop workflow 동작: + +1. `scripts/load_test/stop.sh`가 `environment/load_test`에서 Terraform init을 실행합니다. +2. `restore_stage_dev=true`이면 stage를 dev datasource 구성으로 복구합니다. +3. `destroy_rds=true`이면 Terraform destroy로 load-test RDS와 load-generator EC2를 제거합니다. ## 참고 -- GitHub Actions는 OIDC로 `AWS_ROLE_ARN`을 assume합니다. +- GitHub Actions는 OIDC로 AWS role을 assume합니다. - private submodule checkout에는 `GH_PAT`를 사용합니다. - prod/stage EC2는 `Name` tag로 조회합니다. -- prod/loadtest DB username/password는 Parameter Store에서 읽습니다. -- loadtest RDS security group은 prod/stage API EC2 security group에서 오는 MySQL 접근만 허용합니다. +- prod/load-test DB 계정 정보는 Parameter Store에서 읽습니다. +- load-test RDS 보안 그룹은 prod/stage API EC2 보안 그룹에서 들어오는 MySQL 접근만 허용합니다. diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh index e3a7414..07c362b 100644 --- a/scripts/load_test/start.sh +++ b/scripts/load_test/start.sh @@ -4,17 +4,11 @@ set -euo pipefail TERRAFORM_DIR="environment/load_test" VAR_FILE="../../config/secrets/load_test.tfvars" DATABASE_NAME="" -MIGRATION_PARAMETER_PREFIX="/solid-connection/loadtest/migration" -PROD_DB_USERNAME_PARAMETER="" -PROD_DB_PASSWORD_PARAMETER="" -LOADTEST_DB_USERNAME_PARAMETER="" -LOADTEST_DB_PASSWORD_PARAMETER="" SWITCH_STAGE_TO_LOADTEST="false" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-1800}" SKIP_TERRAFORM_APPLY="false" -SKIP_DATA_COPY="false" usage() { cat <<'EOF' @@ -23,18 +17,12 @@ Usage: scripts/load_test/start.sh [options] Options: --terraform-dir PATH Default: environment/load_test --var-file PATH Default: ../../config/secrets/load_test.tfvars - --prod-db-username-parameter Default: Terraform output prod_db_username_parameter_name - --prod-db-password-parameter Default: Terraform output prod_db_password_parameter_name - --loadtest-db-username-parameter Default: Terraform output load_test_db_username_parameter_name - --loadtest-db-password-parameter Default: Terraform output load_test_db_password_parameter_name --database-name VALUE Default: Terraform output load_test_db_name - --migration-prefix VALUE Default: /solid-connection/loadtest/migration --switch-stage-to-loadtest Restart stage app through SSM with dev,loadtest profiles --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml --ssm-command-timeout-seconds Default: 1800 --skip-terraform-apply - --skip-data-copy -h, --help EOF } @@ -43,18 +31,12 @@ while [[ $# -gt 0 ]]; do case "$1" in --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; --var-file) VAR_FILE="$2"; shift 2 ;; - --prod-db-username-parameter) PROD_DB_USERNAME_PARAMETER="$2"; shift 2 ;; - --prod-db-password-parameter) PROD_DB_PASSWORD_PARAMETER="$2"; shift 2 ;; - --loadtest-db-username-parameter) LOADTEST_DB_USERNAME_PARAMETER="$2"; shift 2 ;; - --loadtest-db-password-parameter) LOADTEST_DB_PASSWORD_PARAMETER="$2"; shift 2 ;; --database-name) DATABASE_NAME="$2"; shift 2 ;; - --migration-prefix) MIGRATION_PARAMETER_PREFIX="$2"; shift 2 ;; --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; - --skip-data-copy) SKIP_DATA_COPY="true"; shift ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1" >&2; usage; exit 1 ;; esac @@ -133,114 +115,19 @@ send_ssm_command() { done } -delete_temp_parameters() { - aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" >/dev/null 2>&1 || true - aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" >/dev/null 2>&1 || true - aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-username" >/dev/null 2>&1 || true - aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-password" >/dev/null 2>&1 || true -} - if [[ "$SKIP_TERRAFORM_APPLY" != "true" ]]; then terraform -chdir="$TERRAFORM_DIR" init terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" fi -prod_instance_id="$(tf_output prod_api_instance_id)" stage_instance_id="$(tf_output stage_api_instance_id)" stage_public_ip="$(tf_output stage_api_public_ip)" -prod_endpoint="$(tf_output prod_rds_endpoint)" -prod_port="$(tf_output prod_rds_port)" loadtest_endpoint="$(tf_output load_test_rds_endpoint)" loadtest_port="$(tf_output load_test_rds_port)" load_generator_instance_id="$(tf_output load_generator_instance_id)" loadtest_db_name="$(tf_output load_test_db_name)" -tf_prod_db_username_parameter="$(tf_output prod_db_username_parameter_name)" -tf_prod_db_password_parameter="$(tf_output prod_db_password_parameter_name)" -tf_loadtest_db_username_parameter="$(tf_output load_test_db_username_parameter_name)" -tf_loadtest_db_password_parameter="$(tf_output load_test_db_password_parameter_name)" DATABASE_NAME="${DATABASE_NAME:-$loadtest_db_name}" -PROD_DB_USERNAME_PARAMETER="${PROD_DB_USERNAME_PARAMETER:-$tf_prod_db_username_parameter}" -PROD_DB_PASSWORD_PARAMETER="${PROD_DB_PASSWORD_PARAMETER:-$tf_prod_db_password_parameter}" -LOADTEST_DB_USERNAME_PARAMETER="${LOADTEST_DB_USERNAME_PARAMETER:-$tf_loadtest_db_username_parameter}" -LOADTEST_DB_PASSWORD_PARAMETER="${LOADTEST_DB_PASSWORD_PARAMETER:-$tf_loadtest_db_password_parameter}" - -if [[ "$SKIP_DATA_COPY" != "true" ]]; then - trap delete_temp_parameters EXIT - - prod_db_username="$(aws ssm get-parameter \ - --name "$PROD_DB_USERNAME_PARAMETER" \ - --query "Parameter.Value" \ - --output text)" - - prod_db_password="$(aws ssm get-parameter \ - --name "$PROD_DB_PASSWORD_PARAMETER" \ - --with-decryption \ - --query "Parameter.Value" \ - --output text)" - - loadtest_db_username="$(aws ssm get-parameter \ - --name "$LOADTEST_DB_USERNAME_PARAMETER" \ - --query "Parameter.Value" \ - --output text)" - - loadtest_db_password="$(aws ssm get-parameter \ - --name "$LOADTEST_DB_PASSWORD_PARAMETER" \ - --with-decryption \ - --query "Parameter.Value" \ - --output text)" - - aws ssm put-parameter \ - --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" \ - --type String \ - --value "$prod_db_username" \ - --overwrite >/dev/null - - aws ssm put-parameter \ - --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" \ - --type SecureString \ - --value "$prod_db_password" \ - --overwrite >/dev/null - - aws ssm put-parameter \ - --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-username" \ - --type String \ - --value "$loadtest_db_username" \ - --overwrite >/dev/null - - aws ssm put-parameter \ - --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-password" \ - --type SecureString \ - --value "$loadtest_db_password" \ - --overwrite >/dev/null - - copy_commands_json="$(jq -cn \ - --arg prefix "$MIGRATION_PARAMETER_PREFIX" \ - --arg prod_endpoint "$prod_endpoint" \ - --arg prod_port "$prod_port" \ - --arg loadtest_endpoint "$loadtest_endpoint" \ - --arg loadtest_port "$loadtest_port" \ - --arg database "$DATABASE_NAME" \ - '{ - commands: [ - "set -euo pipefail", - "export DEBIAN_FRONTEND=noninteractive", - "if ! command -v mysqldump >/dev/null 2>&1 || ! command -v mysql >/dev/null 2>&1; then sudo apt-get update && sudo apt-get install -y mysql-client; fi", - "PROD_USER=$(aws ssm get-parameter --name \($prefix)/prod-db-username --query Parameter.Value --output text)", - "PROD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/prod-db-password --with-decryption --query Parameter.Value --output text)", - "LOAD_USER=$(aws ssm get-parameter --name \($prefix)/loadtest-db-username --query Parameter.Value --output text)", - "LOAD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/loadtest-db-password --with-decryption --query Parameter.Value --output text)", - "DUMP_FILE=/tmp/solid-connection-loadtest-$(date +%Y%m%d%H%M%S).sql.gz", - "trap '\''rm -f \"$DUMP_FILE\"'\'' EXIT", - "MYSQL_PWD=\"$PROD_PASSWORD\" mysqldump --single-transaction --set-gtid-purged=OFF --column-statistics=0 -h \($prod_endpoint) -P \($prod_port) -u \"$PROD_USER\" \($database) | gzip > \"$DUMP_FILE\"", - "MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" -e \"DROP DATABASE IF EXISTS \\\`\($database)\\\`; CREATE DATABASE \\\`\($database)\\\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;\"", - "gunzip -c \"$DUMP_FILE\" | MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" \($database)", - "rm -f \"$DUMP_FILE\"" - ] - }')" - - send_ssm_command "$prod_instance_id" "Copy prod RDS data to load test RDS" "$copy_commands_json" -fi if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then stage_commands_json="$(jq -cn \ From f6e3174e35f743d88977a881e6bd4fd3309fb9ff Mon Sep 17 00:00:00 2001 From: Yeonri Date: Fri, 29 May 2026 01:51:20 +0900 Subject: [PATCH 14/17] =?UTF-8?q?feat:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=8B=9C=ED=81=AC=EB=A6=BF=20=EC=B0=B8?= =?UTF-8?q?=EC=A1=B0=20=EA=B0=B1=EC=8B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 스냅샷 복원 방식에 맞춰 갱신된 secrets submodule 커밋을 참조하도록 수정 --- config/secrets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/secrets b/config/secrets index b11ecad..40fe5a2 160000 --- a/config/secrets +++ b/config/secrets @@ -1 +1 @@ -Subproject commit b11ecadb9329eb4e84b93e98e94d8ab685605caa +Subproject commit 40fe5a23122f5b5fe84c1ecbcec237318ba6358d From ab1135c5e7cc6fa59bbbf09e0820132ddbafe228 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Fri, 29 May 2026 02:21:53 +0900 Subject: [PATCH 15/17] =?UTF-8?q?feat:=20=EB=A0=88=EA=B1=B0=EC=8B=9C=20?= =?UTF-8?q?=EB=B6=80=ED=95=98=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=8A=A4?= =?UTF-8?q?=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 현재 RDS 스냅샷 기반 플로우에서 사용하지 않는 set-load-test.sh를 제거하도록 수정 - k6 동기화 목록과 README에서 레거시 스크립트 참조를 제거하도록 수정 --- config/load-test/k6/script/set-load-test.sh | 64 --------------------- scripts/load_test/README.md | 1 - scripts/load_test/run_k6.sh | 5 +- 3 files changed, 2 insertions(+), 68 deletions(-) delete mode 100644 config/load-test/k6/script/set-load-test.sh diff --git a/config/load-test/k6/script/set-load-test.sh b/config/load-test/k6/script/set-load-test.sh deleted file mode 100644 index b401b2d..0000000 --- a/config/load-test/k6/script/set-load-test.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# 작업 디렉터리 설정 -WORKDIR="/home/ubuntu" - -####################################################################### -# set-load-test.sh -# 사용 예: SQL_FILE_BASENAME=regular.sql ./set-load-test.sh -####################################################################### - -# 0. 필수 값 검증 -SQL_BASENAME="${SQL_FILE_BASENAME:-regular.sql}" -if [[ -z "$SQL_BASENAME" ]]; then - echo "❌ 사용할 덤프 파일이 입력되지 않았습니다." >&2 - echo "SQL_FILE_BASENAME={dump 파일 이름} ./set-load-test.sh 형식으로 인자를 전달해야합니다." >&2 - exit 1 -fi - -SQL_SRC="$WORKDIR/load-test-setting/db/${SQL_BASENAME}" -if [[ ! -f "$SQL_SRC" ]]; then - echo "❌ 덤프 파일이 파일 시스템에 존재하지 않습니다: $SQL_SRC" >&2 - exit 2 -fi - -# 1. 기존 어플리케이션, DB 중지 -docker compose -f "$WORKDIR/solid-connection-dev/docker-compose.dev.yml" down \ - || { echo "❌ 어플리케이션 도커 중지 실패: $WORKDIR/solid-connection-dev/docker-compose.dev.yml" >&2; exit 3; } -docker compose -f "$WORKDIR/mysql/docker-compose.mysql.yml" down \ - || { echo "❌ MySQL 도커 중지 실패: $WORKDIR/mysql/docker-compose.mysql.yml" >&2; exit 4; } - -# 2. 부하 테스트용 DB 실행 -docker compose -f "$WORKDIR/load-test-setting/docker-compose.load-test.yml" up -d \ - || { echo "❌ 부하 테스트용 DB 실행 실패: docker-compose.load-test.yml" >&2; exit 5; } - -# 3. MySQL 준비 대기 (최대 30초) -CONTAINER_NAME="load-test-db" -echo "⏳ MySQL이 준비될 때까지 대기 중 (최대 30초)..." -start_time=$(date +%s) -while ! docker exec "$CONTAINER_NAME" sh -c 'mysqladmin ping -h "127.0.0.1" --silent'; do - elapsed=$(( $(date +%s) - start_time )) - if [[ $elapsed -ge 30 ]]; then - echo "❌ MySQL 준비 시간 초과 (30초)" >&2 - exit 1 - fi - printf "." - sleep 1 -done -echo "✔️ MySQL 준비 완료." - -# 4. dump 주입 -echo "📥 dump 파일 복사 → 컨테이너: $CONTAINER_NAME" -docker cp "$SQL_SRC" "${CONTAINER_NAME}:/tmp/dump.sql" \ - || { echo "❌ dump 복사 실패: $SQL_SRC → $CONTAINER_NAME:/tmp/dump.sql" >&2; exit 6; } - -echo "⚙️ dump 이식" -docker exec -i "$CONTAINER_NAME" sh -c 'mysql -u root -proot < /tmp/dump.sql' \ - || { echo "❌ dump 이식 실패: 컨테이너 $CONTAINER_NAME" >&2; exit 7; } - -# 5. 어플리케이션 다시 실행 -docker compose -f "$WORKDIR/solid-connection-dev/docker-compose.dev.yml" up -d \ - || { echo "❌ 어플리케이션 재시작 실패: $WORKDIR/solid-connection-dev/docker-compose.dev.yml" >&2; exit 8; } - -echo "✅ 부하 테스트용 DB에 연결된 어플리케이션 실행 완료!" diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 7b81f7f..59f0762 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -79,7 +79,6 @@ Run workflow 동작: - `updatePost.json` - `whole-user-flow.js` - `set_up_xk6.sh` -- `script/set-load-test.sh` ## 결과 확인 diff --git a/scripts/load_test/run_k6.sh b/scripts/load_test/run_k6.sh index e2b0477..613c8da 100644 --- a/scripts/load_test/run_k6.sh +++ b/scripts/load_test/run_k6.sh @@ -188,8 +188,7 @@ for relative_path in \ "createPost.json" \ "updatePost.json" \ "whole-user-flow.js" \ - "set_up_xk6.sh" \ - "script/set-load-test.sh"; do + "set_up_xk6.sh"; do sync_file "$load_generator_instance_id" "$load_generator_k6_dir" "$relative_path" done @@ -205,7 +204,7 @@ run_commands_json="$(jq -cn \ commands: [ "set -euo pipefail", "cd \($k6_dir)", - "chmod +x set_up_xk6.sh script/set-load-test.sh", + "chmod +x set_up_xk6.sh", "chown -R ubuntu:ubuntu \($k6_dir)", "if [ ! -x ./k6 ]; then sudo -u ubuntu -H ./set_up_xk6.sh; fi", "sudo -u ubuntu -H env BASE_URL=\($target_base_url | @sh) K6_PROMETHEUS_RW_SERVER_URL=\($prometheus_url | @sh) K6_VUS=\($vus | @sh) K6_ITERATIONS=\($iterations | @sh) K6_MAX_DURATION=\($max_duration | @sh) ./k6 run \($script | @sh)" From ea8688382159d7b28446c168e436a95129144114 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Fri, 29 May 2026 16:09:33 +0900 Subject: [PATCH 16/17] =?UTF-8?q?feat:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EB=A6=AC=EB=B7=B0=20=ED=95=AD=EB=AA=A9=20?= =?UTF-8?q?=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - workflow 입력값을 env로 전달해 shell injection 위험을 줄이도록 수정 - load-generator EC2에 IMDSv2를 강제하도록 수정 - k6 remote-write 출력과 xk6 버전 고정을 추가하도록 수정 - deprecated output과 Prometheus URL 하드코딩을 제거하도록 수정 --- .github/workflows/load-test-run.yml | 20 +++++++++++++------- config/load-test/k6/set_up_xk6.sh | 14 ++++++++------ environment/load_test/main.tf | 6 ++++++ environment/load_test/output.tf | 10 ---------- environment/load_test/variables.tf | 8 ++++---- scripts/load_test/README.md | 5 +++-- scripts/load_test/run_k6.sh | 2 +- 7 files changed, 35 insertions(+), 30 deletions(-) diff --git a/.github/workflows/load-test-run.yml b/.github/workflows/load-test-run.yml index d8675e9..32d9432 100644 --- a/.github/workflows/load-test-run.yml +++ b/.github/workflows/load-test-run.yml @@ -66,19 +66,25 @@ jobs: sudo apt-get install -y jq - name: Run k6 on load generator + env: + VUS: ${{ inputs.vus }} + ITERATIONS: ${{ inputs.iterations }} + MAX_DURATION: ${{ inputs.max_duration }} + TARGET_BASE_URL: ${{ inputs.target_base_url }} + PROMETHEUS_REMOTE_WRITE_URL: ${{ inputs.prometheus_remote_write_url }} run: | args=( - --vus "${{ inputs.vus }}" - --iterations "${{ inputs.iterations }}" - --max-duration "${{ inputs.max_duration }}" + --vus "$VUS" + --iterations "$ITERATIONS" + --max-duration "$MAX_DURATION" ) - if [ -n "${{ inputs.target_base_url }}" ]; then - args+=(--target-base-url "${{ inputs.target_base_url }}") + if [ -n "$TARGET_BASE_URL" ]; then + args+=(--target-base-url "$TARGET_BASE_URL") fi - if [ -n "${{ inputs.prometheus_remote_write_url }}" ]; then - args+=(--prometheus-remote-write-url "${{ inputs.prometheus_remote_write_url }}") + if [ -n "$PROMETHEUS_REMOTE_WRITE_URL" ]; then + args+=(--prometheus-remote-write-url "$PROMETHEUS_REMOTE_WRITE_URL") fi bash scripts/load_test/run_k6.sh "${args[@]}" diff --git a/config/load-test/k6/set_up_xk6.sh b/config/load-test/k6/set_up_xk6.sh index eb1f254..b0edaa5 100644 --- a/config/load-test/k6/set_up_xk6.sh +++ b/config/load-test/k6/set_up_xk6.sh @@ -4,14 +4,16 @@ set -euo pipefail trap 'echo "xk6 setup failed" >&2' ERR -export GO_VERSION=1.22.2 +export GO_VERSION=1.25.9 +export XK6_VERSION=v1.4.3 +export XK6_PROMETHEUS_REMOTE_VERSION=v0.5.1 export BASE_DIR=/home/ubuntu/solid-connection-load-test/k6 export GOROOT=${BASE_DIR}/go export GOPATH=${BASE_DIR}/go-workspace export PATH=$PATH:$GOROOT/bin:$GOPATH/bin export XK6_BIN=${GOPATH}/bin/xk6 -export K6_OUT=xk6-prometheus-rw -export K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL:-http://132.145.83.182:9090/api/v1/write} +export K6_OUT=experimental-prometheus-rw +export K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL:-} export K6_PROMETHEUS_RW_TREND_STATS="${K6_PROMETHEUS_RW_TREND_STATS:-p(90),p(95),p(99),avg,min,max}" { @@ -20,7 +22,7 @@ export K6_PROMETHEUS_RW_TREND_STATS="${K6_PROMETHEUS_RW_TREND_STATS:-p(90),p(95) echo "export GOPATH=${GOPATH}" echo "export PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin" echo "export XK6_BIN=${GOPATH}/bin/xk6" - echo "export K6_OUT=xk6-prometheus-rw" + echo "export K6_OUT=experimental-prometheus-rw" echo "export K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL}" echo "export K6_PROMETHEUS_RW_TREND_STATS=\"${K6_PROMETHEUS_RW_TREND_STATS}\"" } >> ~/.bashrc @@ -39,13 +41,13 @@ rm "go${GO_VERSION}.linux-amd64.tar.gz" echo "Go version: $(go version)" echo "Install xk6" -go install go.k6.io/xk6/cmd/xk6@latest +go install "go.k6.io/xk6/cmd/xk6@${XK6_VERSION}" echo "xk6 installed: ${XK6_BIN}" "$XK6_BIN" --help > /dev/null && echo "xk6 executable is available" echo "Build k6 with Prometheus remote-write output" -"$XK6_BIN" build --with github.com/grafana/xk6-output-prometheus-remote@latest +"$XK6_BIN" build --with "github.com/grafana/xk6-output-prometheus-remote@${XK6_PROMETHEUS_REMOTE_VERSION}" echo "Build complete: $(pwd)/k6" ls -lh ./k6 diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index f68c207..20ae056 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -128,6 +128,12 @@ resource "aws_instance" "load_generator" { associate_public_ip_address = true iam_instance_profile = var.load_generator_instance_profile_name + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + } + root_block_device { volume_size = var.load_generator_root_volume_size volume_type = "gp3" diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf index 2299413..1a58928 100644 --- a/environment/load_test/output.tf +++ b/environment/load_test/output.tf @@ -48,16 +48,6 @@ output "load_test_ssm_parameter_prefix" { value = var.load_test_parameter_prefix } -output "load_test_db_username_parameter_name" { - description = "SSM parameter name containing the load test DB username" - value = var.load_test_db_username_parameter_name -} - -output "load_test_db_password_parameter_name" { - description = "SSM SecureString parameter name containing the load test DB password" - value = var.load_test_db_password_parameter_name -} - output "prod_db_username_parameter_name" { description = "SSM parameter name containing the prod DB username" value = var.prod_db_username_parameter_name diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index d0dc556..1a99863 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -33,14 +33,14 @@ variable "db_name" { } variable "load_test_db_username_parameter_name" { - description = "Deprecated. Load-test datasource credentials are copied from prod datasource parameters." + description = "Deprecated compatibility input. Load-test datasource credentials are copied from prod datasource parameters." type = string default = null nullable = true } variable "load_test_db_password_parameter_name" { - description = "Deprecated. Load-test datasource credentials are copied from prod datasource parameters." + description = "Deprecated compatibility input. Load-test datasource credentials are copied from prod datasource parameters." type = string default = null nullable = true @@ -124,7 +124,7 @@ variable "load_test_target_base_url" { } variable "k6_prometheus_remote_write_url" { - description = "Default Prometheus remote-write URL for k6" + description = "Default Prometheus remote-write URL for k6. Empty disables remote-write unless the workflow input overrides it." type = string - default = "http://132.145.83.182:9090/api/v1/write" + default = "" } diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 59f0762..37fe37d 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -64,6 +64,7 @@ GitHub에서 **Actions > Load Test Run**을 수동 실행합니다. - 선택값입니다. 비워두면 Terraform output `load_test_target_base_url`을 사용합니다. - `prometheus_remote_write_url` - 선택값입니다. 비워두면 Terraform output `k6_prometheus_remote_write_url`을 사용합니다. + - Terraform output도 비어 있으면 Prometheus remote-write 전송은 비활성화됩니다. Run workflow 동작: @@ -84,10 +85,10 @@ Run workflow 동작: 간단한 실행 결과는 **Load Test Run** GitHub Actions 로그에서 확인합니다. -k6 스크립트는 Prometheus remote-write로도 지표를 전송합니다. +k6 스크립트는 remote-write URL이 설정된 경우 Prometheus remote-write로도 지표를 전송합니다. - 기본 remote-write URL은 Terraform output `k6_prometheus_remote_write_url`을 사용합니다. -- workflow 입력값 `prometheus_remote_write_url`로 override할 수 있습니다. +- Terraform 기본값은 비어 있으므로, 전송이 필요하면 Terraform 변수나 workflow 입력값 `prometheus_remote_write_url`로 URL을 넣습니다. - k6 지표에는 요청 수, 실패율, 응답 시간, p90, p95, p99, 평균, 최소, 최대값이 포함됩니다. - API 호출에는 `name`, `testid`, `time` tag가 붙어 endpoint와 실행 시점별로 필터링할 수 있습니다. diff --git a/scripts/load_test/run_k6.sh b/scripts/load_test/run_k6.sh index 613c8da..715015e 100644 --- a/scripts/load_test/run_k6.sh +++ b/scripts/load_test/run_k6.sh @@ -207,7 +207,7 @@ run_commands_json="$(jq -cn \ "chmod +x set_up_xk6.sh", "chown -R ubuntu:ubuntu \($k6_dir)", "if [ ! -x ./k6 ]; then sudo -u ubuntu -H ./set_up_xk6.sh; fi", - "sudo -u ubuntu -H env BASE_URL=\($target_base_url | @sh) K6_PROMETHEUS_RW_SERVER_URL=\($prometheus_url | @sh) K6_VUS=\($vus | @sh) K6_ITERATIONS=\($iterations | @sh) K6_MAX_DURATION=\($max_duration | @sh) ./k6 run \($script | @sh)" + "sudo -u ubuntu -H env BASE_URL=\($target_base_url | @sh) K6_PROMETHEUS_RW_SERVER_URL=\($prometheus_url | @sh) K6_PROMETHEUS_RW_TREND_STATS=\"p(90),p(95),p(99),avg,min,max\" K6_VUS=\($vus | @sh) K6_ITERATIONS=\($iterations | @sh) K6_MAX_DURATION=\($max_duration | @sh) ./k6 run \(if $prometheus_url != \"\" then \"-o experimental-prometheus-rw \" else \"\" end)\($script | @sh)" ] }')" From 23a3e57e9bae32f5bca17922a36b69d1fbfecc2b Mon Sep 17 00:00:00 2001 From: Yeonri Date: Sat, 30 May 2026 19:15:45 +0900 Subject: [PATCH 17/17] =?UTF-8?q?feat:=20=EB=B6=80=ED=95=98=20=EC=83=9D?= =?UTF-8?q?=EC=84=B1=EA=B8=B0=20=EC=8B=A4=ED=96=89=20=ED=9B=84=20=EC=A0=95?= =?UTF-8?q?=EB=A6=AC=20=EC=98=B5=EC=85=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/load-test-run.yml | 18 +++++++++++++ .github/workflows/load-test-start.yml | 1 + environment/load_test/main.tf | 13 ++++++--- environment/load_test/output.tf | 4 +-- environment/load_test/variables.tf | 6 +++++ scripts/load_test/README.md | 38 ++++++++++++++++++--------- scripts/load_test/run_k6.sh | 35 ++++++++++++++++++++++++ scripts/load_test/start.sh | 3 +-- 8 files changed, 98 insertions(+), 20 deletions(-) diff --git a/.github/workflows/load-test-run.yml b/.github/workflows/load-test-run.yml index 32d9432..6ccb7f4 100644 --- a/.github/workflows/load-test-run.yml +++ b/.github/workflows/load-test-run.yml @@ -28,6 +28,16 @@ on: required: false default: "" type: string + destroy_runner: + description: "Destroy the k6 load generator after this run" + required: true + default: true + type: boolean + rebuild_k6: + description: "Rebuild the k6 binary before running" + required: true + default: false + type: boolean permissions: id-token: write @@ -87,4 +97,12 @@ jobs: args+=(--prometheus-remote-write-url "$PROMETHEUS_REMOTE_WRITE_URL") fi + if [ "${{ inputs.destroy_runner }}" != "true" ]; then + args+=(--skip-runner-destroy) + fi + + if [ "${{ inputs.rebuild_k6 }}" = "true" ]; then + args+=(--rebuild-k6) + fi + bash scripts/load_test/run_k6.sh "${args[@]}" diff --git a/.github/workflows/load-test-start.yml b/.github/workflows/load-test-start.yml index c5e02b6..fd1c160 100644 --- a/.github/workflows/load-test-start.yml +++ b/.github/workflows/load-test-start.yml @@ -48,6 +48,7 @@ jobs: - name: Start load test environment run: | export TF_VAR_load_generator_instance_type="c7i.large" + export TF_VAR_create_load_generator="false" if [ -n "${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" ]; then export TF_VAR_load_generator_instance_profile_name="${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index 20ae056..98753c4 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -104,6 +104,8 @@ resource "aws_security_group_rule" "load_test_db_mysql" { } resource "aws_security_group" "load_generator" { + count = var.create_load_generator ? 1 : 0 + name = "sc-load-test-generator-sg" description = "Security group for k6 load generator" vpc_id = data.aws_subnet.stage_api.vpc_id @@ -121,10 +123,12 @@ resource "aws_security_group" "load_generator" { } resource "aws_instance" "load_generator" { + count = var.create_load_generator ? 1 : 0 + ami = data.aws_ami.ubuntu.id instance_type = var.load_generator_instance_type subnet_id = data.aws_instance.stage_api.subnet_id - vpc_security_group_ids = [aws_security_group.load_generator.id] + vpc_security_group_ids = [aws_security_group.load_generator[0].id] associate_public_ip_address = true iam_instance_profile = var.load_generator_instance_profile_name @@ -135,9 +139,10 @@ resource "aws_instance" "load_generator" { } root_block_device { - volume_size = var.load_generator_root_volume_size - volume_type = "gp3" - encrypted = true + volume_size = var.load_generator_root_volume_size + volume_type = "gp3" + encrypted = true + delete_on_termination = true } user_data = <<-EOF diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf index 1a58928..36e2db7 100644 --- a/environment/load_test/output.tf +++ b/environment/load_test/output.tf @@ -60,12 +60,12 @@ output "prod_db_password_parameter_name" { output "load_generator_instance_id" { description = "k6 load generator EC2 instance ID" - value = aws_instance.load_generator.id + value = try(aws_instance.load_generator[0].id, "") } output "load_generator_private_ip" { description = "k6 load generator private IP" - value = aws_instance.load_generator.private_ip + value = try(aws_instance.load_generator[0].private_ip, "") } output "load_generator_k6_dir" { diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index 1a99863..2710ffe 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -99,6 +99,12 @@ variable "load_generator_instance_type" { default = "c7i.xlarge" } +variable "create_load_generator" { + description = "Whether to create the k6 load generator EC2 instance" + type = bool + default = true +} + variable "load_generator_instance_profile_name" { description = "Existing IAM instance profile name for the k6 load generator. It must allow SSM RunCommand." type = string diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 37fe37d..3027b9b 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -5,14 +5,15 @@ 전체 흐름은 다음과 같습니다. 1. **Load Test Start**: 임시 부하 테스트 인프라를 만들고 stage를 준비합니다. -2. **Load Test Run**: 별도 k6 부하 생성 EC2에서 k6를 실행합니다. +2. **Load Test Run**: k6 부하 생성 EC2를 만들고 k6를 실행한 뒤 기본적으로 제거합니다. 3. **Load Test Stop**: stage를 복구하고 임시 부하 테스트 스택을 제거합니다. ## 규칙 - 환경 Terraform에 대해 로컬에서 `terraform apply` 또는 `terraform destroy`를 실행하지 않습니다. - 시작, 실행, 종료는 GitHub Actions에서 수행합니다. -- k6는 stage EC2에서 실행하지 않습니다. 부하 테스트용 Terraform이 생성한 별도 load-generator EC2에서 실행합니다. +- k6는 stage EC2에서 실행하지 않습니다. Run workflow가 생성한 별도 load-generator EC2에서 실행합니다. +- load-generator EC2는 비용 절감을 위해 기본적으로 Run workflow 종료 시 제거합니다. - SSH private key를 사용하지 않습니다. EC2 명령은 SSM RunCommand로 실행합니다. ## 필요한 설정 @@ -44,12 +45,13 @@ Start workflow 동작: 1. GitHub Actions가 `environment/load_test`에서 Terraform apply를 실행합니다. 2. Terraform이 최신 prod RDS 자동 snapshot을 조회합니다. 3. Terraform이 해당 snapshot에서 load-test RDS를 복원합니다. -4. Terraform이 보안 그룹과 `c7i.large` 타입의 k6 load-generator EC2를 생성합니다. -5. Terraform이 load-test datasource 값을 Parameter Store에 기록합니다. +4. Terraform이 load-test datasource 값을 Parameter Store에 기록합니다. - datasource URL은 복원된 load-test RDS endpoint를 사용합니다. - datasource username/password는 prod datasource Parameter Store 값을 사용합니다. -6. `scripts/load_test/start.sh`가 Terraform output에서 필요한 값을 읽습니다. -7. `switch_stage_to_loadtest=true`이면 stage 앱을 `dev,loadtest` profile로 재기동합니다. +5. `scripts/load_test/start.sh`가 Terraform output에서 필요한 값을 읽습니다. +6. `switch_stage_to_loadtest=true`이면 stage 앱을 `dev,loadtest` profile로 재기동합니다. + +Start workflow는 load-generator EC2를 만들지 않습니다. 부하 생성용 EC2는 비용 누수를 막기 위해 Run workflow에서만 생성합니다. ## Load Test Run @@ -65,14 +67,26 @@ GitHub에서 **Actions > Load Test Run**을 수동 실행합니다. - `prometheus_remote_write_url` - 선택값입니다. 비워두면 Terraform output `k6_prometheus_remote_write_url`을 사용합니다. - Terraform output도 비어 있으면 Prometheus remote-write 전송은 비활성화됩니다. +- `destroy_runner`: `true` 또는 `false` + - 기본값은 `true`입니다. + - `true`이면 k6 실행이 끝난 뒤 load-generator EC2를 제거합니다. + - `false`이면 디버깅이나 재실행을 위해 load-generator EC2를 남깁니다. +- `rebuild_k6`: `true` 또는 `false` + - 기본값은 `false`입니다. + - `true`이면 실행 전 기존 k6 binary를 지우고 `set_up_xk6.sh`로 다시 빌드합니다. Run workflow 동작: -1. `scripts/load_test/run_k6.sh`가 Terraform output에서 load-generator EC2 ID와 k6 기본값을 읽습니다. -2. load-generator EC2의 SSM agent가 online 상태가 될 때까지 기다립니다. -3. SSM RunCommand로 k6 파일을 load-generator EC2에 동기화합니다. -4. k6 binary가 없으면 `set_up_xk6.sh`로 Prometheus remote-write 지원이 포함된 k6를 빌드합니다. -5. load-generator EC2에서 `whole-user-flow.js`를 실행합니다. +1. `scripts/load_test/run_k6.sh`가 Terraform target apply로 load-generator EC2와 보안 그룹을 생성합니다. +2. Terraform output에서 load-generator EC2 ID와 k6 기본값을 읽습니다. +3. load-generator EC2의 SSM agent가 online 상태가 될 때까지 기다립니다. +4. SSM RunCommand로 k6 파일을 load-generator EC2에 동기화합니다. +5. 이전 실행에서 남아 있을 수 있는 k6 프로세스를 정리합니다. +6. k6 binary가 없거나 `rebuild_k6=true`이면 `set_up_xk6.sh`로 Prometheus remote-write 지원이 포함된 k6를 빌드합니다. +7. load-generator EC2에서 `whole-user-flow.js`를 실행합니다. +8. `destroy_runner=true`이면 실행 성공/실패와 관계없이 load-generator EC2와 보안 그룹을 제거합니다. + +`destroy_runner=false`로 runner를 남긴 뒤 다시 Run workflow를 실행해도 됩니다. 이 경우 기존 EC2를 재사용하며, k6 파일은 매번 다시 동기화됩니다. 동기화되는 k6 파일: @@ -107,7 +121,7 @@ Stop workflow 동작: 1. `scripts/load_test/stop.sh`가 `environment/load_test`에서 Terraform init을 실행합니다. 2. `restore_stage_dev=true`이면 stage를 dev datasource 구성으로 복구합니다. -3. `destroy_rds=true`이면 Terraform destroy로 load-test RDS와 load-generator EC2를 제거합니다. +3. `destroy_rds=true`이면 Terraform destroy로 load-test RDS와 남아 있는 load-generator EC2를 제거합니다. ## 참고 diff --git a/scripts/load_test/run_k6.sh b/scripts/load_test/run_k6.sh index 715015e..ba0f099 100644 --- a/scripts/load_test/run_k6.sh +++ b/scripts/load_test/run_k6.sh @@ -2,6 +2,7 @@ set -euo pipefail TERRAFORM_DIR="environment/load_test" +VAR_FILE="../../config/secrets/load_test.tfvars" LOCAL_K6_DIR="config/load-test/k6" K6_SCRIPT="whole-user-flow.js" TARGET_BASE_URL="" @@ -10,6 +11,8 @@ K6_VUS="10" K6_ITERATIONS="10" K6_MAX_DURATION="15m" SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-3600}" +DESTROY_RUNNER="true" +REBUILD_K6="false" usage() { cat <<'EOF' @@ -17,6 +20,7 @@ Usage: scripts/load_test/run_k6.sh [options] Options: --terraform-dir PATH Default: environment/load_test + --var-file PATH Default: ../../config/secrets/load_test.tfvars --local-k6-dir PATH Default: config/load-test/k6 --script FILE Default: whole-user-flow.js --target-base-url URL Default: Terraform output load_test_target_base_url @@ -25,6 +29,8 @@ Options: --iterations VALUE Default: 10 --max-duration VALUE Default: 15m --ssm-command-timeout-seconds Default: 3600 + --skip-runner-destroy Keep the k6 load generator after the run + --rebuild-k6 Rebuild the k6 binary before running -h, --help EOF } @@ -32,6 +38,7 @@ EOF while [[ $# -gt 0 ]]; do case "$1" in --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --var-file) VAR_FILE="$2"; shift 2 ;; --local-k6-dir) LOCAL_K6_DIR="$2"; shift 2 ;; --script) K6_SCRIPT="$2"; shift 2 ;; --target-base-url) TARGET_BASE_URL="$2"; shift 2 ;; @@ -40,6 +47,8 @@ while [[ $# -gt 0 ]]; do --iterations) K6_ITERATIONS="$2"; shift 2 ;; --max-duration) K6_MAX_DURATION="$2"; shift 2 ;; --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; + --skip-runner-destroy) DESTROY_RUNNER="false"; shift ;; + --rebuild-k6) REBUILD_K6="true"; shift ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1" >&2; usage; exit 1 ;; esac @@ -61,6 +70,26 @@ tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" } +runner_targets=( + -target=aws_security_group.load_generator + -target=aws_instance.load_generator +) + +destroy_runner() { + local exit_code="$?" + local cleanup_code=0 + + if [[ "$DESTROY_RUNNER" == "true" ]]; then + terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" "${runner_targets[@]}" || cleanup_code="$?" + fi + + if [[ "$exit_code" -ne 0 ]]; then + exit "$exit_code" + fi + + exit "$cleanup_code" +} + send_ssm_command() { local instance_id="$1" local comment="$2" @@ -173,6 +202,9 @@ sync_file() { } terraform -chdir="$TERRAFORM_DIR" init +terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" "${runner_targets[@]}" + +trap destroy_runner EXIT load_generator_instance_id="$(tf_output load_generator_instance_id)" load_generator_k6_dir="$(tf_output load_generator_k6_dir)" @@ -200,12 +232,15 @@ run_commands_json="$(jq -cn \ --arg vus "$K6_VUS" \ --arg iterations "$K6_ITERATIONS" \ --arg max_duration "$K6_MAX_DURATION" \ + --arg rebuild_k6 "$REBUILD_K6" \ '{ commands: [ "set -euo pipefail", "cd \($k6_dir)", + "pkill -f '\''(^|/)k6( |$)'\'' || true", "chmod +x set_up_xk6.sh", "chown -R ubuntu:ubuntu \($k6_dir)", + "if [ \($rebuild_k6 | @sh) = '\''true'\'' ]; then rm -f ./k6; fi", "if [ ! -x ./k6 ]; then sudo -u ubuntu -H ./set_up_xk6.sh; fi", "sudo -u ubuntu -H env BASE_URL=\($target_base_url | @sh) K6_PROMETHEUS_RW_SERVER_URL=\($prometheus_url | @sh) K6_PROMETHEUS_RW_TREND_STATS=\"p(90),p(95),p(99),avg,min,max\" K6_VUS=\($vus | @sh) K6_ITERATIONS=\($iterations | @sh) K6_MAX_DURATION=\($max_duration | @sh) ./k6 run \(if $prometheus_url != \"\" then \"-o experimental-prometheus-rw \" else \"\" end)\($script | @sh)" ] diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh index 07c362b..f7ca2c8 100644 --- a/scripts/load_test/start.sh +++ b/scripts/load_test/start.sh @@ -124,7 +124,6 @@ stage_instance_id="$(tf_output stage_api_instance_id)" stage_public_ip="$(tf_output stage_api_public_ip)" loadtest_endpoint="$(tf_output load_test_rds_endpoint)" loadtest_port="$(tf_output load_test_rds_port)" -load_generator_instance_id="$(tf_output load_generator_instance_id)" loadtest_db_name="$(tf_output load_test_db_name)" DATABASE_NAME="${DATABASE_NAME:-$loadtest_db_name}" @@ -152,6 +151,6 @@ fi echo "Load test environment is ready." echo "RDS endpoint: ${loadtest_endpoint}:${loadtest_port}" -echo "Load generator instance: ${load_generator_instance_id}" +echo "Load generator instance: created by Load Test Run" echo "Stage instance: ${stage_instance_id}" echo "Stage public IP: ${stage_public_ip}"