Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 17 additions & 10 deletions .github/create_and_upload_logs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,33 @@ set -x
sudo chown -R $(whoami):$(whoami) $SUITE/

echo $version > version.log.txt
echo "https://gitlab.com/altinity-qa/clickhouse/cicd/clickhouse-regression/-/pipelines/$GITHUB_RUN_ID" > pipeline_url.log.txt
tfs --debug --no-colors transform nice-new-fails raw.log nice-new-fails.log.txt
tfs --debug --no-colors transform fails raw.log fails.log.txt
tfs --debug --no-colors report results -a "$JOB_REPORT_INDEX" raw.log - $confidential --copyright "Altinity Inc." --logo ./altinity.png | tfs --debug --no-colors document convert > report.html
echo "Re-compress the raw.log"
cat raw.log | xzcat | xz -z -T $(nproc) - > raw.log.2
mv raw.log.2 raw.log

if [[ "$artifacts" == "hetzner" ]]; then
# Avoid set -x expanding secret values into the job log.
{ set +x; } 2>/dev/null
export AWS_ACCESS_KEY_ID="$HETZNER_S3_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$HETZNER_S3_ACCESS_KEY"
export AWS_DEFAULT_REGION="$HETZNER_S3_REGION"
set -x
fi

#Specify whether logs should be uploaded.
if [[ $1 == 1 ]];
then
echo "::notice title=$SUITE$STORAGE $(uname -i) s3 logs and reports::$REPORT_INDEX_URL"
./retry.sh 5 30 aws s3 cp pipeline_url.log.txt $JOB_S3_ROOT/pipeline_url.log.txt --content-type "\"text/plain; charset=utf-8\""
./retry.sh 5 30 aws s3 cp version.log.txt $SUITE_REPORT_BUCKET_PATH/version.log.txt --content-type "\"text/plain; charset=utf-8\""
./retry.sh 5 30 aws s3 cp raw.log $SUITE_REPORT_BUCKET_PATH/raw.log
./retry.sh 5 30 aws s3 cp nice-new-fails.log.txt $SUITE_REPORT_BUCKET_PATH/nice-new-fails.log.txt --content-type "\"text/plain; charset=utf-8\""
./retry.sh 5 30 aws s3 cp fails.log.txt $SUITE_REPORT_BUCKET_PATH/fails.log.txt --content-type "\"text/plain; charset=utf-8\""
./retry.sh 5 30 aws s3 cp report.html $SUITE_REPORT_BUCKET_PATH/report.html
./retry.sh 5 30 aws $aws_endpoint_args s3 cp version.log.txt $SUITE_REPORT_BUCKET_PATH/version.log.txt --content-type "\"text/plain; charset=utf-8\""
./retry.sh 5 30 aws $aws_endpoint_args s3 cp raw.log $SUITE_REPORT_BUCKET_PATH/raw.log
./retry.sh 5 30 aws $aws_endpoint_args s3 cp nice-new-fails.log.txt $SUITE_REPORT_BUCKET_PATH/nice-new-fails.log.txt --content-type "\"text/plain; charset=utf-8\""
./retry.sh 5 30 aws $aws_endpoint_args s3 cp fails.log.txt $SUITE_REPORT_BUCKET_PATH/fails.log.txt --content-type "\"text/plain; charset=utf-8\""
./retry.sh 5 30 aws $aws_endpoint_args s3 cp report.html $SUITE_REPORT_BUCKET_PATH/report.html
sudo rm --recursive --force $SUITE/_instances/*/database/
./retry.sh 5 30 "aws s3 cp --recursive . $SUITE_REPORT_BUCKET_PATH/"' --exclude "*" --include "*/_instances/*.log" --content-type "\"text/plain; charset=utf-8\"" --no-follow-symlinks'
./retry.sh 5 30 "aws $aws_endpoint_args s3 cp --recursive . $SUITE_REPORT_BUCKET_PATH/"' --exclude "*" --include "*/_instances/*.log" --content-type "\"text/plain; charset=utf-8\"" --no-follow-symlinks'
# oauth (and similar suites) collect docker service logs next to the compose
# env dir (envs/<provider>/_service_logs), not at <suite>/_service_logs.
if [[ ! -d "$SUITE/_service_logs" ]]; then
Expand All @@ -34,5 +41,5 @@ then
fi
done
fi
./retry.sh 5 30 "aws s3 cp --recursive $SUITE/_service_logs/ $SUITE_REPORT_BUCKET_PATH/_service_logs/"' --exclude "*" --include "*.log" --content-type "\"text/plain; charset=utf-8\""'
fi
./retry.sh 5 30 "aws $aws_endpoint_args s3 cp --recursive $SUITE/_service_logs/ $SUITE_REPORT_BUCKET_PATH/_service_logs/"' --exclude "*" --include "*.log" --content-type "\"text/plain; charset=utf-8\""'
fi
61 changes: 40 additions & 21 deletions .github/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -87,24 +87,43 @@ fi
echo "::endgroup::"

echo "::group::Generate upload paths..."
if [[ $artifacts == 'internal' ]]; then
artifact_s3_bucket_path="altinity-internal-test-reports"
artifact_s3_dir="clickhouse/$version/$GITHUB_RUN_ID/testflows"
confidential="--confidential"
elif [[ $artifacts == 'public' ]]; then
artifact_s3_bucket_path="altinity-test-reports"
artifact_s3_dir="clickhouse/$version/$GITHUB_RUN_ID/testflows"
confidential=""
elif [[ $artifacts == 'builds' ]]; then
artifact_s3_bucket_path="altinity-build-artifacts"
confidential=""
if [[ $event_name == "pull_request" ]]; then
artifact_s3_dir="PRs/$pr_number/$build_sha/regression"
else # release, push, workflow_dispatch
artifact_s3_dir="REFs/$GITHUB_REF_NAME/$build_sha/regression"
fi

fi
# Bucket layout: hetzner reuses HETZNER_S3_BUCKET; AWS values use legacy buckets.
case "$artifacts" in
hetzner)
artifact_s3_bucket_path="${HETZNER_S3_BUCKET}"
artifact_s3_dir="clickhouse/$version/$GITHUB_RUN_ID/testflows"
confidential="--confidential"
aws_endpoint_args="--endpoint-url https://${HETZNER_S3_REGION}.your-objectstorage.com"
S3_ENDPOINT="https://${HETZNER_S3_REGION}.your-objectstorage.com"
;;
internal)
artifact_s3_bucket_path="altinity-internal-test-reports"
artifact_s3_dir="clickhouse/$version/$GITHUB_RUN_ID/testflows"
confidential="--confidential"
aws_endpoint_args=""
;;
public)
artifact_s3_bucket_path="altinity-test-reports"
artifact_s3_dir="clickhouse/$version/$GITHUB_RUN_ID/testflows"
confidential=""
aws_endpoint_args=""
;;
builds)
artifact_s3_bucket_path="altinity-build-artifacts"
confidential=""
if [[ $event_name == "pull_request" ]]; then
artifact_s3_dir="PRs/$pr_number/$build_sha/regression"
else
artifact_s3_dir="REFs/$GITHUB_REF_NAME/$build_sha/regression"
fi
aws_endpoint_args=""
;;
*)
echo "::error::Unknown artifacts='$artifacts'"
exit 1
;;
esac

if [[ $args == *'--with-analyzer'* ]]; then
analyzer="with_analyzer"
Expand All @@ -125,16 +144,16 @@ else
fi


if [[ -z "$S3_ENDPOINT" ]]; then
if [[ -z "$S3_ENDPOINT" ]]; then
JOB_BUCKET_URL=https://$artifact_s3_bucket_path.s3.amazonaws.com
else
JOB_BUCKET_URL=https://$S3_ENDPOINT/$artifact_s3_bucket_path
JOB_BUCKET_URL=$S3_ENDPOINT/$artifact_s3_bucket_path
fi

echo "confidential=$confidential" >>$GITHUB_ENV
echo "aws_endpoint_args=$aws_endpoint_args" >>$GITHUB_ENV

JOB_REPORT_INDEX=$JOB_BUCKET_URL/index.html#$artifact_s3_dir/
echo "JOB_REPORT_INDEX=$JOB_REPORT_INDEX" >>$GITHUB_ENV

JOB_S3_ROOT=s3://$artifact_s3_bucket_path/$artifact_s3_dir
echo "JOB_S3_ROOT=$JOB_S3_ROOT" >>$GITHUB_ENV
Expand All @@ -150,4 +169,4 @@ echo "SUITE_REPORT_BUCKET_PATH=$SUITE_REPORT_BUCKET_PATH" >>$GITHUB_ENV

echo "::endgroup::"

echo "Browse results at $SUITE_REPORT_INDEX_URL"
echo "Browse results at $SUITE_REPORT_INDEX_URL"
11 changes: 10 additions & 1 deletion .github/upload_results_to_database.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,17 @@

set -x

if [[ "$artifacts" == "hetzner" ]]; then
# Avoid set -x expanding secret values into the job log.
{ set +x; } 2>/dev/null
export AWS_ACCESS_KEY_ID="$HETZNER_S3_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$HETZNER_S3_ACCESS_KEY"
export AWS_DEFAULT_REGION="$HETZNER_S3_REGION"
set -x
fi

if [[ $1 == 1 ]];
then
./retry.sh 5 30 timeout 8m ./.github/upload_results_to_database.py -o nice --log-file raw.log --db-name="gh-data" --db-port=8443 --secure --no-verify --table="clickhouse_regression_results" --log uploader.log || echo "Failed ($?) to upload results to database"
aws s3 cp uploader.log $SUITE_REPORT_BUCKET_PATH/uploader.log
aws $aws_endpoint_args s3 cp uploader.log $SUITE_REPORT_BUCKET_PATH/uploader.log
fi
15 changes: 11 additions & 4 deletions .github/workflows/reusable-suite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,21 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
# Hetzner: used both for hetzner report uploads (artifacts=hetzner, in
# setup.sh/create_and_upload_logs.sh) and for the hetzner test storage
HETZNER_S3_BUCKET: ${{ vars.HETZNER_S3_BUCKET }}
HETZNER_S3_REGION: ${{ vars.HETZNER_S3_REGION }}
HETZNER_S3_KEY_ID: ${{ secrets.HETZNER_S3_KEY_ID }}
HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
HETZNER_S3_URI: "https://${{ vars.HETZNER_S3_REGION }}.your-objectstorage.com/${{ vars.HETZNER_S3_BUCKET }}/"
# Docker credentials
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
# Database credentials
CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}
CHECKS_DATABASE_USER: ${{ secrets.CHECKS_DATABASE_USER }}
CHECKS_DATABASE_PASSWORD: ${{ secrets.CHECKS_DATABASE_PASSWORD }}
# LocalStack token
# LocalStack token
LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
# Python encoding
PYTHONIOENCODING: utf-8
Expand Down Expand Up @@ -168,8 +175,8 @@ jobs:
run: python3
-u ${{ env.SUITE }}/${{ env.SUITE_EXECUTABLE }}
${{ env.REGRESSION_ARGS }}
${{ env.args }} || EXITCODE=$?;
.github/add_link_to_logs.sh;
${{ env.args }} || EXITCODE=$?;
.github/add_link_to_logs.sh;
exit $EXITCODE

- name: 📝 Create and upload logs
Expand All @@ -189,4 +196,4 @@ jobs:

- name: ♻️ Cache sync
if: always()
uses: ./.github/actions/cache-sync
uses: ./.github/actions/cache-sync
8 changes: 8 additions & 0 deletions .github/workflows/run-alter-stress.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,11 @@ on:
description: "Artifact S3 bucket"
type: choice
options:
- hetzner
- internal
- public
- builds
default: hetzner
output_format:
description: "📜 Testflows output style."
type: choice
Expand Down Expand Up @@ -103,6 +107,10 @@ env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
HETZNER_S3_BUCKET: ${{ vars.HETZNER_S3_BUCKET }}
HETZNER_S3_REGION: ${{ vars.HETZNER_S3_REGION }}
HETZNER_S3_KEY_ID: ${{ secrets.HETZNER_S3_KEY_ID }}
HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}
Expand Down
Loading
Loading