Blame cmd/osbuild-image-tests/aws.go

Packit 63bb0d
// +build integration
Packit 63bb0d
Packit 63bb0d
package main
Packit 63bb0d
Packit 63bb0d
import (
Packit 63bb0d
	"encoding/base64"
Packit 63bb0d
	"errors"
Packit 63bb0d
	"fmt"
Packit 63bb0d
	"io/ioutil"
Packit 63bb0d
	"os"
Packit 63bb0d
Packit 63bb0d
	"github.com/aws/aws-sdk-go/aws"
Packit 63bb0d
	"github.com/aws/aws-sdk-go/aws/credentials"
Packit 63bb0d
	"github.com/aws/aws-sdk-go/aws/session"
Packit 63bb0d
	"github.com/aws/aws-sdk-go/service/ec2"
Packit 63bb0d
	"github.com/osbuild/osbuild-composer/internal/upload/awsupload"
Packit 63bb0d
)
Packit 63bb0d
Packit 63bb0d
type awsCredentials struct {
Packit 63bb0d
	AccessKeyId     string
Packit 63bb0d
	SecretAccessKey string
Packit 63bb0d
	Region          string
Packit 63bb0d
	Bucket          string
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// getAWSCredentialsFromEnv gets the credentials from environment variables
Packit 63bb0d
// If none of the environment variables is set, it returns nil.
Packit 63bb0d
// If some but not all environment variables are set, it returns an error.
Packit 63bb0d
func getAWSCredentialsFromEnv() (*awsCredentials, error) {
Packit 63bb0d
	accessKeyId, akExists := os.LookupEnv("AWS_ACCESS_KEY_ID")
Packit 63bb0d
	secretAccessKey, sakExists := os.LookupEnv("AWS_SECRET_ACCESS_KEY")
Packit 63bb0d
	region, regionExists := os.LookupEnv("AWS_REGION")
Packit 63bb0d
	bucket, bucketExists := os.LookupEnv("AWS_BUCKET")
Packit 63bb0d
Packit 63bb0d
	// Workaround Travis security feature. If non of the variables is set, just ignore the test
Packit 63bb0d
	if !akExists && !sakExists && !bucketExists && !regionExists {
Packit 63bb0d
		return nil, nil
Packit 63bb0d
	}
Packit 63bb0d
	// If only one/two of them are not set, then fail
Packit 63bb0d
	if !akExists || !sakExists || !bucketExists || !regionExists {
Packit 63bb0d
		return nil, errors.New("not all required env variables were set")
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return &awsCredentials{
Packit 63bb0d
		AccessKeyId:     accessKeyId,
Packit 63bb0d
		SecretAccessKey: secretAccessKey,
Packit 63bb0d
		Region:          region,
Packit 63bb0d
		Bucket:          bucket,
Packit 63bb0d
	}, nil
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// encodeBase64 encodes string to base64-encoded string
Packit 63bb0d
func encodeBase64(input string) string {
Packit 63bb0d
	return base64.StdEncoding.EncodeToString([]byte(input))
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// createUserData creates cloud-init's user-data that contains user redhat with
Packit 63bb0d
// the specified public key
Packit 63bb0d
func createUserData(publicKeyFile string) (string, error) {
Packit 63bb0d
	publicKey, err := ioutil.ReadFile(publicKeyFile)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return "", fmt.Errorf("cannot read the public key: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	userData := fmt.Sprintf(`#cloud-config
Packit 63bb0d
user: redhat
Packit 63bb0d
ssh_authorized_keys:
Packit 63bb0d
  - %s
Packit 63bb0d
`, string(publicKey))
Packit 63bb0d
Packit 63bb0d
	return userData, nil
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// wrapErrorf returns error constructed using fmt.Errorf from format and any
Packit 63bb0d
// other args. If innerError != nil, it's appended at the end of the new
Packit 63bb0d
// error.
Packit 63bb0d
func wrapErrorf(innerError error, format string, a ...interface{}) error {
Packit 63bb0d
	if innerError != nil {
Packit 63bb0d
		a = append(a, innerError)
Packit 63bb0d
		return fmt.Errorf(format+"\n\ninner error: %#s", a...)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return fmt.Errorf(format, a...)
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// uploadImageToAWS mimics the upload feature of osbuild-composer.
Packit 63bb0d
// It takes an image and an image name and creates an ec2 instance from them.
Packit 63bb0d
// The s3 key is never returned - the same thing is done in osbuild-composer,
Packit 63bb0d
// the user has no way of getting the s3 key.
Packit 63bb0d
func uploadImageToAWS(c *awsCredentials, imagePath string, imageName string) error {
Packit 63bb0d
	uploader, err := awsupload.New(c.Region, c.AccessKeyId, c.SecretAccessKey)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return fmt.Errorf("cannot create aws uploader: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	_, err = uploader.Upload(imagePath, c.Bucket, imageName)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return fmt.Errorf("cannot upload the image: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
	_, err = uploader.Register(imageName, c.Bucket, imageName)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return fmt.Errorf("cannot register the image: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return nil
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// newEC2 creates EC2 struct from given credentials
Packit 63bb0d
func newEC2(c *awsCredentials) (*ec2.EC2, error) {
Packit 63bb0d
	creds := credentials.NewStaticCredentials(c.AccessKeyId, c.SecretAccessKey, "")
Packit 63bb0d
	sess, err := session.NewSession(&aws.Config{
Packit 63bb0d
		Credentials: creds,
Packit 63bb0d
		Region:      aws.String(c.Region),
Packit 63bb0d
	})
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return nil, fmt.Errorf("cannot create aws session: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return ec2.New(sess), nil
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type imageDescription struct {
Packit 63bb0d
	Id         *string
Packit 63bb0d
	SnapshotId *string
Packit 63bb0d
	// this doesn't support multiple snapshots per one image,
Packit 63bb0d
	// because this feature is not supported in composer
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// describeEC2Image searches for EC2 image by its name and returns
Packit 63bb0d
// its id and snapshot id
Packit 63bb0d
func describeEC2Image(e *ec2.EC2, imageName string) (*imageDescription, error) {
Packit 63bb0d
	imageDescriptions, err := e.DescribeImages(&ec2.DescribeImagesInput{
Packit 63bb0d
		Filters: []*ec2.Filter{
Packit 63bb0d
			{
Packit 63bb0d
				Name: aws.String("name"),
Packit 63bb0d
				Values: []*string{
Packit 63bb0d
					aws.String(imageName),
Packit 63bb0d
				},
Packit 63bb0d
			},
Packit 63bb0d
		},
Packit 63bb0d
	})
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return nil, fmt.Errorf("cannot describe the image: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
	imageId := imageDescriptions.Images[0].ImageId
Packit 63bb0d
	snapshotId := imageDescriptions.Images[0].BlockDeviceMappings[0].Ebs.SnapshotId
Packit 63bb0d
Packit 63bb0d
	return &imageDescription{
Packit 63bb0d
		Id:         imageId,
Packit 63bb0d
		SnapshotId: snapshotId,
Packit 63bb0d
	}, nil
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// deleteEC2Image deletes the specified image and its associated snapshot
Packit 63bb0d
func deleteEC2Image(e *ec2.EC2, imageDesc *imageDescription) error {
Packit 63bb0d
	var retErr error
Packit 63bb0d
Packit 63bb0d
	// firstly, deregister the image
Packit 63bb0d
	_, err := e.DeregisterImage(&ec2.DeregisterImageInput{
Packit 63bb0d
		ImageId: imageDesc.Id,
Packit 63bb0d
	})
Packit 63bb0d
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		retErr = wrapErrorf(retErr, "cannot deregister the image: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	// now it's possible to delete the snapshot
Packit 63bb0d
	_, err = e.DeleteSnapshot(&ec2.DeleteSnapshotInput{
Packit 63bb0d
		SnapshotId: imageDesc.SnapshotId,
Packit 63bb0d
	})
Packit 63bb0d
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		retErr = wrapErrorf(retErr, "cannot delete the snapshot: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return retErr
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// withBootedImageInEC2 runs the function f in the context of booted
Packit 63bb0d
// image in AWS EC2
Packit 63bb0d
func withBootedImageInEC2(e *ec2.EC2, imageDesc *imageDescription, publicKey string, f func(address string) error) (retErr error) {
Packit 63bb0d
	// generate user data with given public key
Packit 63bb0d
	userData, err := createUserData(publicKey)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return err
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	// Security group must be now generated, because by default
Packit 63bb0d
	// all traffic to EC2 instance is filtered.
Packit 63bb0d
Packit 63bb0d
	securityGroupName, err := generateRandomString("osbuild-image-tests-security-group-")
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return fmt.Errorf("cannot generate a random name for the image: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	// Firstly create a security group
Packit 63bb0d
	securityGroup, err := e.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
Packit 63bb0d
		GroupName:   aws.String(securityGroupName),
Packit 63bb0d
		Description: aws.String("image-tests-security-group"),
Packit 63bb0d
	})
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return fmt.Errorf("cannot create a new security group: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	defer func() {
Packit 63bb0d
		_, err = e.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
Packit 63bb0d
			GroupId: securityGroup.GroupId,
Packit 63bb0d
		})
Packit 63bb0d
Packit 63bb0d
		if err != nil {
Packit 63bb0d
			retErr = wrapErrorf(retErr, "cannot delete the security group: %#v", err)
Packit 63bb0d
		}
Packit 63bb0d
	}()
Packit 63bb0d
Packit 63bb0d
	// Authorize incoming SSH connections.
Packit 63bb0d
	_, err = e.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
Packit 63bb0d
		CidrIp:     aws.String("0.0.0.0/0"),
Packit 63bb0d
		GroupId:    securityGroup.GroupId,
Packit 63bb0d
		FromPort:   aws.Int64(22),
Packit 63bb0d
		ToPort:     aws.Int64(22),
Packit 63bb0d
		IpProtocol: aws.String("tcp"),
Packit 63bb0d
	})
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return fmt.Errorf("canot add a rule to the security group: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	// Finally, run the instance from the given image and with the created security group
Packit 63bb0d
	res, err := e.RunInstances(&ec2.RunInstancesInput{
Packit 63bb0d
		MaxCount:         aws.Int64(1),
Packit 63bb0d
		MinCount:         aws.Int64(1),
Packit 63bb0d
		ImageId:          imageDesc.Id,
Packit 63bb0d
		InstanceType:     aws.String("t3.micro"),
Packit 63bb0d
		SecurityGroupIds: []*string{securityGroup.GroupId},
Packit 63bb0d
		UserData:         aws.String(encodeBase64(userData)),
Packit 63bb0d
	})
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return fmt.Errorf("cannot create a new instance: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	describeInstanceInput := &ec2.DescribeInstancesInput{
Packit 63bb0d
		InstanceIds: []*string{
Packit 63bb0d
			res.Instances[0].InstanceId,
Packit 63bb0d
		},
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	defer func() {
Packit 63bb0d
		// We need to terminate the instance now and wait until the termination is done.
Packit 63bb0d
		// Otherwise, it wouldn't be possible to delete the image.
Packit 63bb0d
		_, err = e.TerminateInstances(&ec2.TerminateInstancesInput{
Packit 63bb0d
			InstanceIds: []*string{
Packit 63bb0d
				res.Instances[0].InstanceId,
Packit 63bb0d
			},
Packit 63bb0d
		})
Packit 63bb0d
		if err != nil {
Packit 63bb0d
			retErr = wrapErrorf(retErr, "cannot terminate the instance: %#v", err)
Packit 63bb0d
			return
Packit 63bb0d
		}
Packit 63bb0d
Packit 63bb0d
		err = e.WaitUntilInstanceTerminated(describeInstanceInput)
Packit 63bb0d
		if err != nil {
Packit 63bb0d
			retErr = wrapErrorf(retErr, "waiting for the instance termination failed: %#v", err)
Packit 63bb0d
		}
Packit 63bb0d
	}()
Packit 63bb0d
Packit 63bb0d
	// The instance has no IP address yet. It's assigned when the instance
Packit 63bb0d
	// is in the state "EXISTS". However, in this state the instance is not
Packit 63bb0d
	// much usable, therefore wait until "RUNNING" state, in which the instance
Packit 63bb0d
	// actually can do something useful for us.
Packit 63bb0d
	err = e.WaitUntilInstanceRunning(describeInstanceInput)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return fmt.Errorf("waiting for the instance to be running failed: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	// By describing the instance, we can get the ip address.
Packit 63bb0d
	out, err := e.DescribeInstances(describeInstanceInput)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return fmt.Errorf("cannot describe the instance: %#v", err)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return f(*out.Reservations[0].Instances[0].PublicIpAddress)
Packit 63bb0d
}