def cron_string = BRANCH_NAME == "main" ? "@daily" : "" pipeline { agent none triggers { cron(cron_string) } environment { AWS_REGION = "us-east-2" AWS_BUCKET = "imagebuilder-jenkins-testing-use2" BUILD_CAUSE = detect_build_cause() } options { timestamps() ansiColor('xterm') // Cancel the pipeline if it runs for more than three hours. timeout( time: 12, unit: "HOURS" ) } stages { stage("Prepare 🤔") { agent { label "schutzbot" } options { // Don't checkout the git repository here. It just clogs // up the Jenkins disk space and does nothing for us. skipDefaultCheckout() } steps { sh ( label: "Get environment variables", script: "env | sort" ) } } stage("Mock build 👷🏻") { when { expression { return env.BUILD_CAUSE != 'cron'; } } // Halt the pipeline immediately if a single mock build fails. // A failure to build an RPM is serious and must be // investigated. failFast true parallel { stage('F32') { agent { label "f32cloudbase && x86_64 && aws" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') } steps { sh "schutzbot/ci_details.sh" retry(3) { sh "schutzbot/mockbuild.sh" } } } stage('F33') { agent { label "f33cloudbase && x86_64 && aws" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') } steps { sh "schutzbot/ci_details.sh" retry(3) { sh "schutzbot/mockbuild.sh" } } } stage('F33 aarch64') { agent { label "f33cloudbase && aarch64 && aws" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') } steps { sh "schutzbot/ci_details.sh" retry(3) { sh "schutzbot/mockbuild.sh" } } } stage('F34') { agent { label "f34cloudbase && x86_64 && aws" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') } steps { sh "schutzbot/ci_details.sh" retry(3) { sh "schutzbot/mockbuild.sh" } } } stage('F34 aarch64') { agent { label "f34cloudbase && aarch64 && aws" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') } steps { sh "schutzbot/ci_details.sh" retry(3) { sh "schutzbot/mockbuild.sh" } } } stage('EL8') { agent { label "rhel8cloudbase && x86_64 && aws" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') } steps { sh "schutzbot/ci_details.sh" retry(3) { sh "schutzbot/mockbuild.sh" } } } stage('EL8.4') { agent { label "rhel84cloudbase && x86_64 && psi" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') RHEL84_NIGHTLY_REPO = credentials('rhel84-nightly-repo') } steps { sh "schutzbot/ci_details.sh" retry(3) { sh "schutzbot/mockbuild.sh" } } } stage('CS8') { agent { label "cs8cloudbase && x86_64 && aws" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') } steps { sh "schutzbot/ci_details.sh" retry(3) { sh "schutzbot/mockbuild.sh" } } } } } stage("Prepare EL8 nightly 🤔") { agent { label "rhel8cloudbase && x86_64 && psi" } when { expression { return env.BUILD_CAUSE == 'cron'; } } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { // don't build, use RPMs from the nightly trees sh "schutzbot/prepare-rhel-nightly.sh" stash ( includes: 'rhel8nightly.repo', name: 'rhel8nightly' ) stash ( includes: 'rhel-8.json', name: 'rhel8json' ) stash ( includes: 'rhel-8-beta.json', name: 'rhel8betajson' ) stash ( includes: 'COMPOSE_ID', name: 'compose_id' ) } } stage("Container build - x86_64") { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f33cloudbase && x86_64 && aws" } steps { sh "schutzbot/containerbuild.sh" } } stage("Testing 🍌") { parallel { stage('F32 Base') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f32cloudbase && x86_64 && aws" } environment { TEST_TYPE = "base" } steps { run_tests('base') } post { always { preserve_logs('fedora32-base') } } } stage('F32 Image') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f32cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') DISTRO_CODE = "fedora32" } steps { run_tests('image') } post { always { preserve_logs('fedora32-image') sh ( label: "Run cloud cleaner just in case something failed", script: "schutzbot/run_cloud_cleaner.sh" ) } } } stage('F32 Integration') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f32cloudbase && x86_64 && aws" } environment { TEST_TYPE = "integration" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') } steps { run_tests('integration') } post { always { preserve_logs('fedora32-integration') } } } stage('F32 OSTree') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f32cloudbase && psi && x86_64" } steps { run_tests('ostree') } post { always { preserve_logs('fedora32-ostree') } } } stage('F32 New OSTree') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f32cloudbase && psi && x86_64" } steps { run_tests('ostree-ng') } post { always { preserve_logs('fedora32-ostree-ng') } } } stage('F32: koji-osbuild') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f32cloudbase && x86_64 && aws" } steps { run_project_tests('koji-osbuild', 'integration.sh') } post { always { preserve_logs('fedora32-revdep-koji-osbuild') } } } stage('F33 Base') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f33cloudbase && x86_64 && aws" } environment { TEST_TYPE = "base" } steps { run_tests('base') } post { always { preserve_logs('fedora33-base') } } } stage('F33 Image') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f33cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') DISTRO_CODE = "fedora33" } steps { run_tests('image') } post { always { preserve_logs('fedora33-image') sh ( label: "Run cloud cleaner just in case something failed", script: "schutzbot/run_cloud_cleaner.sh" ) } } } stage('F33 Integration') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f33cloudbase && x86_64 && aws" } environment { TEST_TYPE = "integration" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') } steps { run_tests('integration') } post { always { preserve_logs('fedora33-integration') } } } stage('F33 OSTree') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f33cloudbase && psi && x86_64" } steps { run_tests('ostree') } post { always { preserve_logs('fedora33-ostree') } } } stage('F33 New OSTree') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f33cloudbase && psi && x86_64" } steps { run_tests('ostree-ng') } post { always { preserve_logs('fedora33-ostree-ng') } } } stage('F33 aarch64 Base') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f33cloudbase && aarch64 && aws" } environment { TEST_TYPE = "base" } steps { run_tests('base') } post { always { preserve_logs('fedora33-aarch64-base') } } } stage('F33 aarch64 Image') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "f33cloudbase && aarch64 && aws" } environment { TEST_TYPE = "image" AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') DISTRO_CODE = "fedora33" } steps { run_tests('image') } post { always { preserve_logs('fedora33-aarch64-image') sh ( label: "Run cloud cleaner just in case something failed", script: "schutzbot/run_cloud_cleaner.sh" ) } } } stage('EL8 Base') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "rhel8cloudbase && x86_64 && psi" } environment { TEST_TYPE = "base" RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') } steps { run_tests('base') } post { always { preserve_logs('rhel8-base') } } } stage('EL8 Image') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "rhel8cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') VCENTER_CREDS = credentials('vmware-vcenter-credentials') DISTRO_CODE = "rhel8" } steps { run_tests('image') } post { always { preserve_logs('rhel8-image') sh ( label: "Run cloud cleaner just in case something failed", script: "schutzbot/run_cloud_cleaner.sh" ) } } } stage('EL8 Integration') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "rhel8cloudbase && x86_64 && psi" } environment { TEST_TYPE = "integration" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') VCENTER_CREDS = credentials('vmware-vcenter-credentials') } steps { run_tests('integration') } post { always { preserve_logs('rhel8-integration') } } } stage('EL8 OSTree') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "rhel8cloudbase && psi && x86_64" } steps { run_tests('ostree') } post { always { preserve_logs('rhel8-ostree') } } } stage('EL8 New OSTree') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "rhel8cloudbase && psi && x86_64" } steps { run_tests('ostree-ng') } post { always { preserve_logs('rhel8-ostree-ng') } } } stage('EL8: koji-osbuild') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "rhel8cloudbase && psi && x86_64" } steps { run_project_tests('koji-osbuild', 'integration.sh') } post { always { preserve_logs('rhel8-revdep-koji-osbuild') } } } stage('EL8.4 Base') { agent { label "rhel84cloudbase && x86_64 && psi" } environment { TEST_TYPE = "base" } steps { run_tests('base') } post { always { preserve_logs('rhel84-base') } } } stage('EL8.4 Image') { agent { label "rhel84cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') DISTRO_CODE = "rhel84" } steps { run_tests('image') } post { always { preserve_logs('rhel84-image') sh ( label: "Run cloud cleaner just in case something failed", script: "schutzbot/run_cloud_cleaner.sh" ) } } } stage('EL8.4 Integration') { agent { label "rhel84cloudbase && x86_64 && psi" } environment { TEST_TYPE = "integration" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') VCENTER_CREDS = credentials('vmware-vcenter-credentials') } steps { run_tests('integration') } post { always { preserve_logs('rhel84-integration') } } } stage('EL8.4 OSTree') { agent { label "rhel84cloudbase && psi && x86_64" } steps { run_tests('ostree') } post { always { preserve_logs('rhel84-ostree') } } } stage('EL8.4 New OSTree') { agent { label "rhel84cloudbase && psi && x86_64" } steps { run_tests('ostree-ng') } post { always { preserve_logs('rhel84-ostree-ng') } } } stage('CS8 Base') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "cs8cloudbase && x86_64 && aws" } environment { TEST_TYPE = "base" } steps { run_tests('base') } post { always { preserve_logs('cs8-base') } } } stage('CS8 Image') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "cs8cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') DISTRO_CODE = "centos-stream8" } steps { run_tests('image') } post { always { preserve_logs('cs8-image') sh ( label: "Run cloud cleaner just in case something failed", script: "schutzbot/run_cloud_cleaner.sh" ) } } } stage('CS8 Integration') { when { expression { return env.BUILD_CAUSE != 'cron'; } } agent { label "cs8cloudbase && x86_64 && psi" } environment { TEST_TYPE = "integration" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') } steps { run_tests('integration') } post { always { preserve_logs('cs8-integration') } } } } } } post { success { node('schutzbot') { script { if (env.BUILD_CAUSE == 'cron') { unstash 'compose_id' def composeId = readFile "${env.WORKSPACE}/COMPOSE_ID" telegramSend """💚 CI passed for osbuild-composer ${composeId} ${env.BUILD_URL}, CC: [@atodorov_kiwi](tg://user?id=1047018883) """ } else if (env.BRANCH_NAME == 'main') { telegramSend "💚 CI passed for osbuild-composer main branch ${env.BUILD_URL}" } } } } unsuccessful { node('schutzbot') { script { if (env.BUILD_CAUSE == 'cron') { unstash 'compose_id' def composeId = readFile "${env.WORKSPACE}/COMPOSE_ID" telegramSend """💣 CI failed for osbuild-composer ${composeId} ${env.BUILD_URL}, CC: [@atodorov_kiwi](tg://user?id=1047018883) """ } else if (env.BRANCH_NAME == 'main') { telegramSend "💣 CI failed for osbuild-composer main branch ${env.BUILD_URL}" } } } } } } // Set up a function to hold the steps needed to run the tests so we don't // need to copy/paste the same lines over and over above. void run_tests(test_type) { // unstash files but avoid failure if they don't exist // b/c these files are available only for nightly builds try { unstash 'rhel8nightly' unstash 'rhel8json' unstash 'rhel8betajson' } catch (err) { echo "ERROR during unstash: ${err}" } // Get CI machine details. sh ( label: "Get CI machine details", script: "schutzbot/ci_details.sh" ) // Deploy the Image Builder packages and services. sh ( label: "Deploy", script: "schutzbot/deploy.sh" ) // Run the base tests. if (test_type == 'base') { sh ( label: "Base tests", script: "/usr/libexec/tests/osbuild-composer/base_tests.sh" ) } if (test_type == 'image') { sh ( label: "Image tests", script: "/usr/libexec/tests/osbuild-composer/image_tests.sh" ) } if (test_type == 'ostree') { sh ( label: "OSTree tests", script: "/usr/libexec/tests/osbuild-composer/ostree.sh" ) } if (test_type == 'ostree-ng') { sh ( label: "New OSTree tests", script: "/usr/libexec/tests/osbuild-composer/ostree-ng.sh" ) } if (test_type == 'integration') { // Run Koji tests. sh ( label: "Koji tests", script: "/usr/libexec/tests/osbuild-composer/koji.sh" ) // Run the libvirt test. sh ( label: "Integration test: libvirt", script: "/usr/libexec/tests/osbuild-composer/libvirt.sh" ) // Run the AWS test. sh ( label: "Integration test: AWS", script: "/usr/libexec/tests/osbuild-composer/aws.sh" ) // Run the API test. sh ( label: "Integration test: API", script: "/usr/libexec/tests/osbuild-composer/api.sh" ) if (env.VCENTER_CREDS) { // Run the VMWare test. sh ( label: "Integration test: VMWare", script: "/usr/libexec/tests/osbuild-composer/vmware.sh" ) } } } // Similar to run_tests, but with a more general signature: allows setting the // project whose -tests package to install, and the name of the test to execute void run_project_tests(project, test) { sh "schutzbot/ci_details.sh" sh "schutzbot/deploy.sh ${project}" sh "/usr/libexec/tests/${project}/${test}" } // Move logs to a unique location and tell Jenkins to capture them on success // or failure. void preserve_logs(test_slug) { // Save the systemd journal. sh "journalctl --boot > systemd-journald.log" // Make a directory for the log files and move the logs there. sh "mkdir ${test_slug} && mv *.log *.jpg ${test_slug}/ || true" // The workspace directory is not used everywhere, tests use temporary directory under /tmp/logs. sh "mkdir -p ${test_slug} && find /tmp/logs/ -name '*.log' -exec mv {} ${test_slug}/ \\; || true" // Artifact the logs. archiveArtifacts ( allowEmptyArchive: true, artifacts: "${test_slug}/*.log,${test_slug}/*.jpg" ) } void detect_build_cause() { def buildCause = currentBuild.getBuildCauses().get(0) if ( buildCause.shortDescription == 'Started by timer' && env.BRANCH_NAME == 'main') { return "cron" } return }