Blame cmd/osbuild-worker/jobimpl-osbuild.go

Packit Service 509fd4
package main
Packit Service 509fd4
Packit Service 509fd4
import (
Packit Service 509fd4
	"crypto/tls"
Packit Service 509fd4
	"fmt"
Packit Service 509fd4
	"io/ioutil"
Packit Service 509fd4
	"log"
Packit Service 509fd4
	"net/http"
Packit Service 509fd4
	"net/url"
Packit Service 509fd4
	"os"
Packit Service 509fd4
	"path"
Packit Service 509fd4
	"time"
Packit Service 509fd4
Packit Service 509fd4
	"github.com/google/uuid"
Packit Service 509fd4
	"github.com/osbuild/osbuild-composer/internal/common"
Packit Service 509fd4
	"github.com/osbuild/osbuild-composer/internal/distro"
Packit Service 15f37d
	osbuild "github.com/osbuild/osbuild-composer/internal/osbuild1"
Packit Service 509fd4
	"github.com/osbuild/osbuild-composer/internal/target"
Packit Service 509fd4
	"github.com/osbuild/osbuild-composer/internal/upload/awsupload"
Packit Service 509fd4
	"github.com/osbuild/osbuild-composer/internal/upload/azure"
Packit Service 509fd4
	"github.com/osbuild/osbuild-composer/internal/upload/koji"
Packit Service 509fd4
	"github.com/osbuild/osbuild-composer/internal/upload/vmware"
Packit Service 509fd4
	"github.com/osbuild/osbuild-composer/internal/worker"
Packit Service 509fd4
)
Packit Service 509fd4
Packit Service 509fd4
type OSBuildJobImpl struct {
Packit Service 509fd4
	Store       string
Packit Service bcdfb1
	Output      string
Packit Service 509fd4
	KojiServers map[string]koji.GSSAPICredentials
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
func packageMetadataToSignature(pkg osbuild.RPMPackageMetadata) *string {
Packit Service 509fd4
	if pkg.SigGPG != "" {
Packit Service 509fd4
		return &pkg.SigGPG
Packit Service 509fd4
	} else if pkg.SigPGP != "" {
Packit Service 509fd4
		return &pkg.SigPGP
Packit Service 509fd4
	}
Packit Service 509fd4
	return nil
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
func osbuildStagesToRPMs(stages []osbuild.StageResult) []koji.RPM {
Packit Service 509fd4
	rpms := make([]koji.RPM, 0)
Packit Service 509fd4
	for _, stage := range stages {
Packit Service 509fd4
		switch metadata := stage.Metadata.(type) {
Packit Service 509fd4
		case *osbuild.RPMStageMetadata:
Packit Service 509fd4
			for _, pkg := range metadata.Packages {
Packit Service 509fd4
				rpms = append(rpms, koji.RPM{
Packit Service 509fd4
					Type:      "rpm",
Packit Service 509fd4
					Name:      pkg.Name,
Packit Service 509fd4
					Epoch:     pkg.Epoch,
Packit Service 509fd4
					Version:   pkg.Version,
Packit Service 509fd4
					Release:   pkg.Release,
Packit Service 509fd4
					Arch:      pkg.Arch,
Packit Service 509fd4
					Sigmd5:    pkg.SigMD5,
Packit Service 509fd4
					Signature: packageMetadataToSignature(pkg),
Packit Service 509fd4
				})
Packit Service 509fd4
			}
Packit Service 509fd4
		default:
Packit Service 509fd4
			continue
Packit Service 509fd4
		}
Packit Service 509fd4
	}
Packit Service 509fd4
	return rpms
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
func (impl *OSBuildJobImpl) Run(job worker.Job) error {
Packit Service bcdfb1
	outputDirectory, err := ioutil.TempDir(impl.Output, job.Id().String()+"-*")
Packit Service 509fd4
	if err != nil {
Packit Service 509fd4
		return fmt.Errorf("error creating temporary output directory: %v", err)
Packit Service 509fd4
	}
Packit Service 509fd4
	defer func() {
Packit Service 509fd4
		err := os.RemoveAll(outputDirectory)
Packit Service 509fd4
		if err != nil {
Packit Service 509fd4
			log.Printf("Error removing temporary output directory (%s): %v", outputDirectory, err)
Packit Service 509fd4
		}
Packit Service 509fd4
	}()
Packit Service 509fd4
Packit Service 509fd4
	var args worker.OSBuildJob
Packit Service 509fd4
	err = job.Args(&args)
Packit Service 509fd4
	if err != nil {
Packit Service 509fd4
		return err
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 15f37d
	exports := args.Exports
Packit Service 15f37d
	if len(exports) == 0 {
Packit Service 15f37d
		// job did not define exports, likely coming from an older version of composer
Packit Service 15f37d
		// fall back to default "assembler"
Packit Service 15f37d
		exports = []string{"assembler"}
Packit Service 15f37d
	} else if len(exports) > 1 {
Packit Service 15f37d
		// this worker only supports returning one (1) export
Packit Service 15f37d
		return fmt.Errorf("at most one build artifact can be exported")
Packit Service 15f37d
	}
Packit Service 15f37d
Packit Service 509fd4
	start_time := time.Now()
Packit Service 509fd4
Packit Service 15f37d
	osbuildOutput, err := RunOSBuild(args.Manifest, impl.Store, outputDirectory, exports, os.Stderr)
Packit Service 509fd4
	if err != nil {
Packit Service 509fd4
		return err
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 509fd4
	end_time := time.Now()
Packit Service 509fd4
Packit Service bcdfb1
	streamOptimizedPath := ""
Packit Service bcdfb1
Packit Service 15f37d
	// NOTE: Currently OSBuild supports multiple exports, but this isn't used
Packit Service 15f37d
	// by any of the image types and it can't be specified during the request.
Packit Service 15f37d
	// Use the first (and presumably only) export for the imagePath.
Packit Service 15f37d
	exportPath := exports[0]
Packit Service 509fd4
	if osbuildOutput.Success && args.ImageName != "" {
Packit Service 509fd4
		var f *os.File
Packit Service 15f37d
		imagePath := path.Join(outputDirectory, exportPath, args.ImageName)
Packit Service 509fd4
		if args.StreamOptimized {
Packit Service 509fd4
			f, err = vmware.OpenAsStreamOptimizedVmdk(imagePath)
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				return err
Packit Service 509fd4
			}
Packit Service bcdfb1
			streamOptimizedPath = f.Name()
Packit Service 509fd4
		} else {
Packit Service 509fd4
			f, err = os.Open(imagePath)
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				return err
Packit Service 509fd4
			}
Packit Service 509fd4
		}
Packit Service 509fd4
		err = job.UploadArtifact(args.ImageName, f)
Packit Service 509fd4
		if err != nil {
Packit Service 509fd4
			return err
Packit Service 509fd4
		}
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 509fd4
	var r []error
Packit Service 509fd4
Packit Service 509fd4
	for _, t := range args.Targets {
Packit Service 509fd4
		switch options := t.Options.(type) {
Packit Service 509fd4
		case *target.LocalTargetOptions:
Packit Service 509fd4
			if !osbuildOutput.Success {
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
			var f *os.File
Packit Service 15f37d
			imagePath := path.Join(outputDirectory, exportPath, options.Filename)
Packit Service 509fd4
			if options.StreamOptimized {
Packit Service 509fd4
				f, err = vmware.OpenAsStreamOptimizedVmdk(imagePath)
Packit Service 509fd4
				if err != nil {
Packit Service 509fd4
					r = append(r, err)
Packit Service 509fd4
					continue
Packit Service 509fd4
				}
Packit Service 509fd4
			} else {
Packit Service 509fd4
				f, err = os.Open(imagePath)
Packit Service 509fd4
				if err != nil {
Packit Service 509fd4
					r = append(r, err)
Packit Service 509fd4
					continue
Packit Service 509fd4
				}
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			err = job.UploadArtifact(options.Filename, f)
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service bcdfb1
		case *target.VMWareTargetOptions:
Packit Service bcdfb1
			if !osbuildOutput.Success {
Packit Service bcdfb1
				continue
Packit Service bcdfb1
			}
Packit Service bcdfb1
Packit Service bcdfb1
			credentials := vmware.Credentials{
Packit Service bcdfb1
				Username:   options.Username,
Packit Service bcdfb1
				Password:   options.Password,
Packit Service bcdfb1
				Host:       options.Host,
Packit Service bcdfb1
				Cluster:    options.Cluster,
Packit Service bcdfb1
				Datacenter: options.Datacenter,
Packit Service bcdfb1
				Datastore:  options.Datastore,
Packit Service bcdfb1
			}
Packit Service bcdfb1
Packit Service bcdfb1
			tempDirectory, err := ioutil.TempDir(impl.Output, job.Id().String()+"-vmware-*")
Packit Service bcdfb1
			if err != nil {
Packit Service bcdfb1
				r = append(r, err)
Packit Service bcdfb1
				continue
Packit Service bcdfb1
			}
Packit Service bcdfb1
Packit Service bcdfb1
			defer func() {
Packit Service bcdfb1
				err := os.RemoveAll(tempDirectory)
Packit Service bcdfb1
				if err != nil {
Packit Service bcdfb1
					log.Printf("Error removing temporary directory for vmware symlink(%s): %v", tempDirectory, err)
Packit Service bcdfb1
				}
Packit Service bcdfb1
			}()
Packit Service bcdfb1
Packit Service bcdfb1
			// create a symlink so that uploaded image has the name specified by user
Packit Service bcdfb1
			imageName := t.ImageName + ".vmdk"
Packit Service bcdfb1
			imagePath := path.Join(tempDirectory, imageName)
Packit Service bcdfb1
			err = os.Symlink(streamOptimizedPath, imagePath)
Packit Service bcdfb1
			if err != nil {
Packit Service bcdfb1
				r = append(r, err)
Packit Service bcdfb1
				continue
Packit Service bcdfb1
			}
Packit Service bcdfb1
Packit Service bcdfb1
			err = vmware.UploadImage(credentials, imagePath)
Packit Service bcdfb1
			if err != nil {
Packit Service bcdfb1
				r = append(r, err)
Packit Service bcdfb1
				continue
Packit Service bcdfb1
			}
Packit Service 509fd4
Packit Service 509fd4
		case *target.AWSTargetOptions:
Packit Service 509fd4
			if !osbuildOutput.Success {
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
			a, err := awsupload.New(options.Region, options.AccessKeyID, options.SecretAccessKey)
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			key := options.Key
Packit Service 509fd4
			if key == "" {
Packit Service 509fd4
				key = uuid.New().String()
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			_, err = a.Upload(path.Join(outputDirectory, options.Filename), options.Bucket, key)
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			/* TODO: communicate back the AMI */
Packit Service 509fd4
			_, err = a.Register(t.ImageName, options.Bucket, key, options.ShareWithAccounts, common.CurrentArch())
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
		case *target.AzureTargetOptions:
Packit Service 509fd4
			if !osbuildOutput.Success {
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
			credentials := azure.Credentials{
Packit Service 509fd4
				StorageAccount:   options.StorageAccount,
Packit Service 509fd4
				StorageAccessKey: options.StorageAccessKey,
Packit Service 509fd4
			}
Packit Service 509fd4
			metadata := azure.ImageMetadata{
Packit Service 509fd4
				ContainerName: options.Container,
Packit Service 509fd4
				ImageName:     t.ImageName,
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			const azureMaxUploadGoroutines = 4
Packit Service 509fd4
			err := azure.UploadImage(
Packit Service 509fd4
				credentials,
Packit Service 509fd4
				metadata,
Packit Service 509fd4
				path.Join(outputDirectory, options.Filename),
Packit Service 509fd4
				azureMaxUploadGoroutines,
Packit Service 509fd4
			)
Packit Service 509fd4
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
		case *target.KojiTargetOptions:
Packit Service 509fd4
			// Koji for some reason needs TLS renegotiation enabled.
Packit Service 509fd4
			// Clone the default http transport and enable renegotiation.
Packit Service 509fd4
			transport := http.DefaultTransport.(*http.Transport).Clone()
Packit Service 509fd4
			transport.TLSClientConfig = &tls.Config{
Packit Service 509fd4
				Renegotiation: tls.RenegotiateOnceAsClient,
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			kojiServer, _ := url.Parse(options.Server)
Packit Service 509fd4
			creds, exists := impl.KojiServers[kojiServer.Hostname()]
Packit Service 509fd4
			if !exists {
Packit Service 509fd4
				r = append(r, fmt.Errorf("Koji server has not been configured: %s", kojiServer.Hostname()))
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			k, err := koji.NewFromGSSAPI(options.Server, &creds, transport)
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			defer func() {
Packit Service 509fd4
				err := k.Logout()
Packit Service 509fd4
				if err != nil {
Packit Service 509fd4
					log.Printf("koji logout failed: %v", err)
Packit Service 509fd4
				}
Packit Service 509fd4
			}()
Packit Service 509fd4
Packit Service 509fd4
			if !osbuildOutput.Success {
Packit Service 509fd4
				err = k.CGFailBuild(int(options.BuildID), options.Token)
Packit Service 509fd4
				if err != nil {
Packit Service 509fd4
					log.Printf("CGFailBuild failed: %v", err)
Packit Service 509fd4
				}
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			f, err := os.Open(path.Join(outputDirectory, options.Filename))
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			hash, filesize, err := k.Upload(f, options.UploadDirectory, options.KojiFilename)
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			hostOS, err := distro.GetRedHatRelease()
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			build := koji.ImageBuild{
Packit Service 509fd4
				BuildID:   options.BuildID,
Packit Service 509fd4
				TaskID:    options.TaskID,
Packit Service 509fd4
				Name:      options.Name,
Packit Service 509fd4
				Version:   options.Version,
Packit Service 509fd4
				Release:   options.Release,
Packit Service 509fd4
				StartTime: start_time.Unix(),
Packit Service 509fd4
				EndTime:   end_time.Unix(),
Packit Service 509fd4
			}
Packit Service 509fd4
			buildRoots := []koji.BuildRoot{
Packit Service 509fd4
				{
Packit Service 509fd4
					ID: 1,
Packit Service 509fd4
					Host: koji.Host{
Packit Service 509fd4
						Os:   hostOS,
Packit Service 509fd4
						Arch: common.CurrentArch(),
Packit Service 509fd4
					},
Packit Service 509fd4
					ContentGenerator: koji.ContentGenerator{
Packit Service 509fd4
						Name:    "osbuild",
Packit Service 509fd4
						Version: "0", // TODO: put the correct version here
Packit Service 509fd4
					},
Packit Service 509fd4
					Container: koji.Container{
Packit Service 509fd4
						Type: "none",
Packit Service 509fd4
						Arch: common.CurrentArch(),
Packit Service 509fd4
					},
Packit Service 509fd4
					Tools: []koji.Tool{},
Packit Service 509fd4
					RPMs:  osbuildStagesToRPMs(osbuildOutput.Build.Stages),
Packit Service 509fd4
				},
Packit Service 509fd4
			}
Packit Service 509fd4
			output := []koji.Image{
Packit Service 509fd4
				{
Packit Service 509fd4
					BuildRootID:  1,
Packit Service 509fd4
					Filename:     options.KojiFilename,
Packit Service 509fd4
					FileSize:     uint64(filesize),
Packit Service 509fd4
					Arch:         common.CurrentArch(),
Packit Service 509fd4
					ChecksumType: "md5",
Packit Service 509fd4
					MD5:          hash,
Packit Service 509fd4
					Type:         "image",
Packit Service 509fd4
					RPMs:         osbuildStagesToRPMs(osbuildOutput.Stages),
Packit Service 509fd4
					Extra: koji.ImageExtra{
Packit Service 509fd4
						Info: koji.ImageExtraInfo{
Packit Service 509fd4
							Arch: "noarch",
Packit Service 509fd4
						},
Packit Service 509fd4
					},
Packit Service 509fd4
				},
Packit Service 509fd4
			}
Packit Service 509fd4
Packit Service 509fd4
			_, err = k.CGImport(build, buildRoots, output, options.UploadDirectory, options.Token)
Packit Service 509fd4
			if err != nil {
Packit Service 509fd4
				r = append(r, err)
Packit Service 509fd4
				continue
Packit Service 509fd4
			}
Packit Service 509fd4
		default:
Packit Service 509fd4
			r = append(r, fmt.Errorf("invalid target type"))
Packit Service 509fd4
		}
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 509fd4
	var targetErrors []string
Packit Service 509fd4
	for _, err := range r {
Packit Service 509fd4
		targetErrors = append(targetErrors, err.Error())
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service bcdfb1
	var uploadstatus string = "failure"
Packit Service bcdfb1
	if len(targetErrors) == 0 {
Packit Service bcdfb1
		uploadstatus = "success"
Packit Service bcdfb1
	}
Packit Service bcdfb1
Packit Service 509fd4
	err = job.Update(&worker.OSBuildJobResult{
Packit Service 509fd4
		Success:       osbuildOutput.Success && len(targetErrors) == 0,
Packit Service 509fd4
		OSBuildOutput: osbuildOutput,
Packit Service 509fd4
		TargetErrors:  targetErrors,
Packit Service bcdfb1
		UploadStatus:  uploadstatus,
Packit Service 509fd4
	})
Packit Service 509fd4
	if err != nil {
Packit Service 509fd4
		return fmt.Errorf("Error reporting job result: %v", err)
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 509fd4
	return nil
Packit Service 509fd4
}