|
Packit Service |
15f37d |
def cron_string = BRANCH_NAME == "rhel-8.4.0" ? "@daily" : ""
|
|
Packit Service |
bcdfb1 |
|
|
Packit |
63bb0d |
pipeline {
|
|
Packit |
63bb0d |
agent none
|
|
Packit |
63bb0d |
|
|
Packit Service |
bcdfb1 |
triggers {
|
|
Packit Service |
bcdfb1 |
cron(cron_string)
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
|
|
Packit |
63bb0d |
environment {
|
|
Packit |
63bb0d |
AWS_REGION = "us-east-2"
|
|
Packit |
63bb0d |
AWS_BUCKET = "imagebuilder-jenkins-testing-use2"
|
|
Packit Service |
bcdfb1 |
BUILD_CAUSE = detect_build_cause()
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
options {
|
|
Packit |
63bb0d |
timestamps()
|
|
Packit |
63bb0d |
ansiColor('xterm')
|
|
Packit |
63bb0d |
// Cancel the pipeline if it runs for more than three hours.
|
|
Packit |
63bb0d |
timeout(
|
|
Packit Service |
3a6627 |
time: 12,
|
|
Packit |
63bb0d |
unit: "HOURS"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
stages {
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
stage("Prepare 🤔") {
|
|
Packit |
63bb0d |
agent { label "schutzbot" }
|
|
Packit |
63bb0d |
options {
|
|
Packit |
63bb0d |
// Don't checkout the git repository here. It just clogs
|
|
Packit |
63bb0d |
// up the Jenkins disk space and does nothing for us.
|
|
Packit |
63bb0d |
skipDefaultCheckout()
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
steps {
|
|
Packit |
63bb0d |
sh (
|
|
Packit |
63bb0d |
label: "Get environment variables",
|
|
Packit |
63bb0d |
script: "env | sort"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
stage("Mock build 👷🏻") {
|
|
Packit Service |
bcdfb1 |
when {
|
|
Packit Service |
bcdfb1 |
expression {
|
|
Packit Service |
bcdfb1 |
return env.BUILD_CAUSE != 'cron';
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
|
|
Packit |
63bb0d |
// Halt the pipeline immediately if a single mock build fails.
|
|
Packit |
63bb0d |
// A failure to build an RPM is serious and must be
|
|
Packit |
63bb0d |
// investigated.
|
|
Packit |
63bb0d |
failFast true
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
parallel {
|
|
Packit Service |
509fd4 |
stage('EL8.4') {
|
|
Packit Service |
509fd4 |
agent { label "rhel84cloudbase && x86_64 && psi" }
|
|
Packit |
63bb0d |
environment {
|
|
Packit |
63bb0d |
AWS_CREDS = credentials('aws-credentials-osbuildci')
|
|
Packit Service |
509fd4 |
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
|
|
Packit Service |
509fd4 |
RHEL84_NIGHTLY_REPO = credentials('rhel84-nightly-repo')
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
steps {
|
|
Packit |
63bb0d |
sh "schutzbot/ci_details.sh"
|
|
Packit |
63bb0d |
retry(3) {
|
|
Packit |
63bb0d |
sh "schutzbot/mockbuild.sh"
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit Service |
15f37d |
stage("Prepare EL8 internal 🤔") {
|
|
Packit Service |
bcdfb1 |
agent { label "rhel8cloudbase && x86_64 && psi" }
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
bcdfb1 |
when {
|
|
Packit Service |
bcdfb1 |
expression {
|
|
Packit Service |
bcdfb1 |
return env.BUILD_CAUSE == 'cron';
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
bcdfb1 |
environment {
|
|
Packit Service |
bcdfb1 |
AWS_CREDS = credentials('aws-credentials-osbuildci')
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
steps {
|
|
Packit Service |
15f37d |
// don't build, use RPMs from the internal trees
|
|
Packit Service |
15f37d |
sh "schutzbot/prepare-rhel-internal.sh"
|
|
Packit Service |
bcdfb1 |
stash (
|
|
Packit Service |
15f37d |
includes: 'rhel8internal.repo',
|
|
Packit Service |
15f37d |
name: 'rhel8internal'
|
|
Packit Service |
bcdfb1 |
)
|
|
Packit Service |
bcdfb1 |
stash (
|
|
Packit Service |
bcdfb1 |
includes: 'rhel-8.json',
|
|
Packit Service |
bcdfb1 |
name: 'rhel8json'
|
|
Packit Service |
bcdfb1 |
)
|
|
Packit Service |
bcdfb1 |
stash (
|
|
Packit Service |
bcdfb1 |
includes: 'rhel-8-beta.json',
|
|
Packit Service |
bcdfb1 |
name: 'rhel8betajson'
|
|
Packit Service |
bcdfb1 |
)
|
|
Packit Service |
bcdfb1 |
stash (
|
|
Packit Service |
bcdfb1 |
includes: 'COMPOSE_ID',
|
|
Packit Service |
bcdfb1 |
name: 'compose_id'
|
|
Packit Service |
bcdfb1 |
)
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
3a6627 |
stage("Container build - x86_64") {
|
|
Packit Service |
bcdfb1 |
when {
|
|
Packit Service |
bcdfb1 |
expression {
|
|
Packit Service |
bcdfb1 |
return env.BUILD_CAUSE != 'cron';
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
3a6627 |
agent { label "f33cloudbase && x86_64 && aws" }
|
|
Packit Service |
3a6627 |
steps {
|
|
Packit Service |
3a6627 |
sh "schutzbot/containerbuild.sh"
|
|
Packit Service |
3a6627 |
}
|
|
Packit Service |
3a6627 |
}
|
|
Packit Service |
3a6627 |
|
|
Packit |
63bb0d |
stage("Testing 🍌") {
|
|
Packit |
63bb0d |
parallel {
|
|
Packit Service |
509fd4 |
stage('EL8.4 Base') {
|
|
Packit Service |
509fd4 |
agent { label "rhel84cloudbase && x86_64 && psi" }
|
|
Packit |
63bb0d |
environment {
|
|
Packit |
63bb0d |
TEST_TYPE = "base"
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
steps {
|
|
Packit |
63bb0d |
run_tests('base')
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
post {
|
|
Packit |
63bb0d |
always {
|
|
Packit Service |
509fd4 |
preserve_logs('rhel84-base')
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit Service |
509fd4 |
stage('EL8.4 Image') {
|
|
Packit Service |
509fd4 |
agent { label "rhel84cloudbase && psi && x86_64" }
|
|
Packit |
63bb0d |
environment {
|
|
Packit |
63bb0d |
TEST_TYPE = "image"
|
|
Packit |
63bb0d |
AWS_CREDS = credentials('aws-credentials-osbuildci')
|
|
Packit Service |
509fd4 |
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
|
|
Packit |
63bb0d |
AZURE_CREDS = credentials('azure')
|
|
Packit |
63bb0d |
OPENSTACK_CREDS = credentials("psi-openstack-creds")
|
|
Packit |
63bb0d |
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
|
|
Packit Service |
509fd4 |
DISTRO_CODE = "rhel84"
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
steps {
|
|
Packit |
63bb0d |
run_tests('image')
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
post {
|
|
Packit |
63bb0d |
always {
|
|
Packit Service |
509fd4 |
preserve_logs('rhel84-image')
|
|
Packit Service |
509fd4 |
sh (
|
|
Packit Service |
509fd4 |
label: "Run cloud cleaner just in case something failed",
|
|
Packit Service |
509fd4 |
script: "schutzbot/run_cloud_cleaner.sh"
|
|
Packit Service |
509fd4 |
)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit Service |
509fd4 |
stage('EL8.4 Integration') {
|
|
Packit Service |
509fd4 |
agent { label "rhel84cloudbase && x86_64 && psi" }
|
|
Packit |
63bb0d |
environment {
|
|
Packit |
63bb0d |
TEST_TYPE = "integration"
|
|
Packit |
63bb0d |
AWS_CREDS = credentials('aws-credentials-osbuildci')
|
|
Packit Service |
509fd4 |
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
|
|
Packit Service |
509fd4 |
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
|
|
Packit Service |
bcdfb1 |
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
steps {
|
|
Packit |
63bb0d |
run_tests('integration')
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
post {
|
|
Packit |
63bb0d |
always {
|
|
Packit Service |
509fd4 |
preserve_logs('rhel84-integration')
|
|
Packit Service |
509fd4 |
}
|
|
Packit Service |
509fd4 |
}
|
|
Packit Service |
509fd4 |
}
|
|
Packit Service |
509fd4 |
stage('EL8.4 OSTree') {
|
|
Packit Service |
509fd4 |
agent { label "rhel84cloudbase && psi && x86_64" }
|
|
Packit Service |
509fd4 |
steps {
|
|
Packit Service |
509fd4 |
run_tests('ostree')
|
|
Packit Service |
509fd4 |
}
|
|
Packit Service |
509fd4 |
post {
|
|
Packit Service |
509fd4 |
always {
|
|
Packit Service |
509fd4 |
preserve_logs('rhel84-ostree')
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit Service |
bcdfb1 |
stage('EL8.4 New OSTree') {
|
|
Packit Service |
bcdfb1 |
agent { label "rhel84cloudbase && psi && x86_64" }
|
|
Packit Service |
bcdfb1 |
steps {
|
|
Packit Service |
bcdfb1 |
run_tests('ostree-ng')
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
post {
|
|
Packit Service |
bcdfb1 |
always {
|
|
Packit Service |
bcdfb1 |
preserve_logs('rhel84-ostree-ng')
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
post {
|
|
Packit |
63bb0d |
success {
|
|
Packit |
63bb0d |
node('schutzbot') {
|
|
Packit |
63bb0d |
script {
|
|
Packit Service |
bcdfb1 |
if (env.BUILD_CAUSE == 'cron') {
|
|
Packit Service |
bcdfb1 |
unstash 'compose_id'
|
|
Packit Service |
bcdfb1 |
def composeId = readFile "${env.WORKSPACE}/COMPOSE_ID"
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
bcdfb1 |
telegramSend """💚 CI passed for osbuild-composer ${composeId} ${env.BUILD_URL},
|
|
Packit Service |
bcdfb1 |
CC: [@atodorov_kiwi](tg://user?id=1047018883)
|
|
Packit Service |
bcdfb1 |
"""
|
|
Packit Service |
bcdfb1 |
} else if (env.BRANCH_NAME == 'main') {
|
|
Packit Service |
509fd4 |
telegramSend "💚 CI passed for osbuild-composer main branch ${env.BUILD_URL}"
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
unsuccessful {
|
|
Packit |
63bb0d |
node('schutzbot') {
|
|
Packit |
63bb0d |
script {
|
|
Packit Service |
bcdfb1 |
if (env.BUILD_CAUSE == 'cron') {
|
|
Packit Service |
bcdfb1 |
unstash 'compose_id'
|
|
Packit Service |
bcdfb1 |
def composeId = readFile "${env.WORKSPACE}/COMPOSE_ID"
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
15f37d |
telegramSend """💣 CI failed for osbuild-composer rhel-8.4.0 branch ${composeId} ${env.BUILD_URL},
|
|
Packit Service |
bcdfb1 |
CC: [@atodorov_kiwi](tg://user?id=1047018883)
|
|
Packit Service |
bcdfb1 |
"""
|
|
Packit Service |
bcdfb1 |
} else if (env.BRANCH_NAME == 'main') {
|
|
Packit Service |
509fd4 |
telegramSend "💣 CI failed for osbuild-composer main branch ${env.BUILD_URL}"
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Set up a function to hold the steps needed to run the tests so we don't
|
|
Packit |
63bb0d |
// need to copy/paste the same lines over and over above.
|
|
Packit |
63bb0d |
void run_tests(test_type) {
|
|
Packit Service |
bcdfb1 |
// unstash files but avoid failure if they don't exist
|
|
Packit Service |
15f37d |
// b/c these files are available only for internal builds
|
|
Packit Service |
bcdfb1 |
try {
|
|
Packit Service |
15f37d |
unstash 'rhel8internal'
|
|
Packit Service |
bcdfb1 |
unstash 'rhel8json'
|
|
Packit Service |
bcdfb1 |
unstash 'rhel8betajson'
|
|
Packit Service |
bcdfb1 |
} catch (err) {
|
|
Packit Service |
bcdfb1 |
echo "ERROR during unstash: ${err}"
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Get CI machine details.
|
|
Packit |
63bb0d |
sh (
|
|
Packit |
63bb0d |
label: "Get CI machine details",
|
|
Packit |
63bb0d |
script: "schutzbot/ci_details.sh"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Deploy the Image Builder packages and services.
|
|
Packit |
63bb0d |
sh (
|
|
Packit |
63bb0d |
label: "Deploy",
|
|
Packit |
63bb0d |
script: "schutzbot/deploy.sh"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Run the base tests.
|
|
Packit |
63bb0d |
if (test_type == 'base') {
|
|
Packit |
63bb0d |
sh (
|
|
Packit |
63bb0d |
label: "Base tests",
|
|
Packit Service |
509fd4 |
script: "/usr/libexec/tests/osbuild-composer/base_tests.sh"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if (test_type == 'image') {
|
|
Packit |
63bb0d |
sh (
|
|
Packit |
63bb0d |
label: "Image tests",
|
|
Packit Service |
509fd4 |
script: "/usr/libexec/tests/osbuild-composer/image_tests.sh"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit Service |
509fd4 |
if (test_type == 'ostree') {
|
|
Packit |
63bb0d |
sh (
|
|
Packit Service |
509fd4 |
label: "OSTree tests",
|
|
Packit Service |
509fd4 |
script: "/usr/libexec/tests/osbuild-composer/ostree.sh"
|
|
Packit |
63bb0d |
)
|
|
Packit Service |
509fd4 |
}
|
|
Packit |
63bb0d |
|
|
Packit Service |
bcdfb1 |
if (test_type == 'ostree-ng') {
|
|
Packit Service |
bcdfb1 |
sh (
|
|
Packit Service |
bcdfb1 |
label: "New OSTree tests",
|
|
Packit Service |
bcdfb1 |
script: "/usr/libexec/tests/osbuild-composer/ostree-ng.sh"
|
|
Packit Service |
bcdfb1 |
)
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
509fd4 |
if (test_type == 'integration') {
|
|
Packit Service |
509fd4 |
// Run Koji tests.
|
|
Packit |
63bb0d |
sh (
|
|
Packit Service |
509fd4 |
label: "Koji tests",
|
|
Packit Service |
509fd4 |
script: "/usr/libexec/tests/osbuild-composer/koji.sh"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
|
|
Packit Service |
509fd4 |
// Run the libvirt test.
|
|
Packit |
63bb0d |
sh (
|
|
Packit Service |
509fd4 |
label: "Integration test: libvirt",
|
|
Packit Service |
509fd4 |
script: "/usr/libexec/tests/osbuild-composer/libvirt.sh"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Run the AWS test.
|
|
Packit |
63bb0d |
sh (
|
|
Packit |
63bb0d |
label: "Integration test: AWS",
|
|
Packit Service |
509fd4 |
script: "/usr/libexec/tests/osbuild-composer/aws.sh"
|
|
Packit Service |
509fd4 |
)
|
|
Packit Service |
509fd4 |
|
|
Packit Service |
509fd4 |
// Run the API test.
|
|
Packit Service |
509fd4 |
sh (
|
|
Packit Service |
509fd4 |
label: "Integration test: API",
|
|
Packit Service |
509fd4 |
script: "/usr/libexec/tests/osbuild-composer/api.sh"
|
|
Packit |
63bb0d |
)
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
bcdfb1 |
if (env.VCENTER_CREDS) {
|
|
Packit Service |
bcdfb1 |
// Run the VMWare test.
|
|
Packit Service |
bcdfb1 |
sh (
|
|
Packit Service |
bcdfb1 |
label: "Integration test: VMWare",
|
|
Packit Service |
bcdfb1 |
script: "/usr/libexec/tests/osbuild-composer/vmware.sh"
|
|
Packit Service |
bcdfb1 |
)
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit |
63bb0d |
}
|
|
Packit Service |
509fd4 |
}
|
|
Packit |
63bb0d |
|
|
Packit Service |
509fd4 |
// Similar to run_tests, but with a more general signature: allows setting the
|
|
Packit Service |
509fd4 |
// project whose -tests package to install, and the name of the test to execute
|
|
Packit Service |
509fd4 |
void run_project_tests(project, test) {
|
|
Packit Service |
509fd4 |
sh "schutzbot/ci_details.sh"
|
|
Packit Service |
509fd4 |
sh "schutzbot/deploy.sh ${project}"
|
|
Packit Service |
509fd4 |
sh "/usr/libexec/tests/${project}/${test}"
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Move logs to a unique location and tell Jenkins to capture them on success
|
|
Packit |
63bb0d |
// or failure.
|
|
Packit |
63bb0d |
void preserve_logs(test_slug) {
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Save the systemd journal.
|
|
Packit |
63bb0d |
sh "journalctl --boot > systemd-journald.log"
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Make a directory for the log files and move the logs there.
|
|
Packit |
63bb0d |
sh "mkdir ${test_slug} && mv *.log *.jpg ${test_slug}/ || true"
|
|
Packit Service |
509fd4 |
// The workspace directory is not used everywhere, tests use temporary directory under /tmp/logs.
|
|
Packit Service |
509fd4 |
sh "mkdir -p ${test_slug} && find /tmp/logs/ -name '*.log' -exec mv {} ${test_slug}/ \\; || true"
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Artifact the logs.
|
|
Packit |
63bb0d |
archiveArtifacts (
|
|
Packit |
63bb0d |
allowEmptyArchive: true,
|
|
Packit |
63bb0d |
artifacts: "${test_slug}/*.log,${test_slug}/*.jpg"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
}
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
bcdfb1 |
void detect_build_cause() {
|
|
Packit Service |
bcdfb1 |
def buildCause = currentBuild.getBuildCauses().get(0)
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
15f37d |
if ( buildCause.shortDescription == 'Started by timer' ) {
|
|
Packit Service |
bcdfb1 |
return "cron"
|
|
Packit Service |
bcdfb1 |
}
|
|
Packit Service |
bcdfb1 |
|
|
Packit Service |
bcdfb1 |
return
|
|
Packit Service |
bcdfb1 |
}
|