Blame internal/upload/koji/koji.go

Packit 63bb0d
package koji
Packit 63bb0d
Packit 63bb0d
import (
Packit 63bb0d
	"bytes"
Packit 63bb0d
	"crypto/md5"
Packit 63bb0d
	"encoding/json"
Packit Service 509fd4
	"errors"
Packit 63bb0d
	"fmt"
Packit 63bb0d
	"hash/adler32"
Packit 63bb0d
	"io"
Packit 63bb0d
	"io/ioutil"
Packit 63bb0d
	"net/http"
Packit 63bb0d
	"net/url"
Packit Service 509fd4
	"os"
Packit 63bb0d
Packit 63bb0d
	"github.com/kolo/xmlrpc"
Packit Service 509fd4
	"github.com/ubccr/kerby/khttp"
Packit 63bb0d
)
Packit 63bb0d
Packit 63bb0d
type Koji struct {
Packit Service 509fd4
	xmlrpc    *xmlrpc.Client
Packit Service 509fd4
	server    string
Packit Service 509fd4
	transport http.RoundTripper
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
type TypeInfo struct {
Packit Service 509fd4
	Image struct{} `json:"image"`
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
type ImageBuildExtra struct {
Packit Service 509fd4
	TypeInfo TypeInfo `json:"typeinfo"`
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
type ImageBuild struct {
Packit Service 509fd4
	BuildID   uint64          `json:"build_id"`
Packit Service 509fd4
	TaskID    uint64          `json:"task_id"`
Packit Service 509fd4
	Name      string          `json:"name"`
Packit Service 509fd4
	Version   string          `json:"version"`
Packit Service 509fd4
	Release   string          `json:"release"`
Packit Service 509fd4
	Source    string          `json:"source"`
Packit Service 509fd4
	StartTime int64           `json:"start_time"`
Packit Service 509fd4
	EndTime   int64           `json:"end_time"`
Packit Service 509fd4
	Extra     ImageBuildExtra `json:"extra"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type Host struct {
Packit 63bb0d
	Os   string `json:"os"`
Packit 63bb0d
	Arch string `json:"arch"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type ContentGenerator struct {
Packit Service 509fd4
	Name    string `json:"name"` // Must be 'osbuild'.
Packit 63bb0d
	Version string `json:"version"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type Container struct {
Packit 63bb0d
	Type string `json:"type"`
Packit 63bb0d
	Arch string `json:"arch"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type Tool struct {
Packit 63bb0d
	Name    string `json:"name"`
Packit 63bb0d
	Version string `json:"version"`
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
type RPM struct {
Packit 63bb0d
	Type      string  `json:"type"` // must be 'rpm'
Packit 63bb0d
	Name      string  `json:"name"`
Packit 63bb0d
	Version   string  `json:"version"`
Packit 63bb0d
	Release   string  `json:"release"`
Packit Service 509fd4
	Epoch     *string `json:"epoch,omitempty"`
Packit 63bb0d
	Arch      string  `json:"arch"`
Packit 63bb0d
	Sigmd5    string  `json:"sigmd5"`
Packit 63bb0d
	Signature *string `json:"signature"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type BuildRoot struct {
Packit 63bb0d
	ID               uint64           `json:"id"`
Packit 63bb0d
	Host             Host             `json:"host"`
Packit 63bb0d
	ContentGenerator ContentGenerator `json:"content_generator"`
Packit 63bb0d
	Container        Container        `json:"container"`
Packit 63bb0d
	Tools            []Tool           `json:"tools"`
Packit Service 509fd4
	RPMs             []RPM            `json:"components"`
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
type ImageExtraInfo struct {
Packit 63bb0d
	// TODO: Ideally this is where the pipeline would be passed.
Packit 63bb0d
	Arch string `json:"arch"` // TODO: why?
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
type ImageExtra struct {
Packit Service 509fd4
	Info ImageExtraInfo `json:"image"`
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
type Image struct {
Packit Service 509fd4
	BuildRootID  uint64     `json:"buildroot_id"`
Packit Service 509fd4
	Filename     string     `json:"filename"`
Packit Service 509fd4
	FileSize     uint64     `json:"filesize"`
Packit Service 509fd4
	Arch         string     `json:"arch"`
Packit Service 509fd4
	ChecksumType string     `json:"checksum_type"` // must be 'md5'
Packit Service 509fd4
	MD5          string     `json:"checksum"`
Packit Service 509fd4
	Type         string     `json:"type"`
Packit Service 509fd4
	RPMs         []RPM      `json:"components"`
Packit Service 509fd4
	Extra        ImageExtra `json:"extra"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type Metadata struct {
Packit 63bb0d
	MetadataVersion int         `json:"metadata_version"` // must be '0'
Packit Service 509fd4
	ImageBuild      ImageBuild  `json:"build"`
Packit 63bb0d
	BuildRoots      []BuildRoot `json:"buildroots"`
Packit Service 509fd4
	Images          []Image     `json:"output"`
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
type CGInitBuildResult struct {
Packit Service 509fd4
	BuildID int    `xmlrpc:"build_id"`
Packit Service 509fd4
	Token   string `xmlrpc:"token"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type CGImportResult struct {
Packit 63bb0d
	BuildID int `xmlrpc:"build_id"`
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
type GSSAPICredentials struct {
Packit Service 509fd4
	Principal string
Packit Service 509fd4
	KeyTab    string
Packit Service 509fd4
}
Packit 63bb0d
Packit Service 509fd4
type loginReply struct {
Packit Service 509fd4
	SessionID  int64  `xmlrpc:"session-id"`
Packit Service 509fd4
	SessionKey string `xmlrpc:"session-key"`
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
func newKoji(server string, transport http.RoundTripper, reply loginReply) (*Koji, error) {
Packit Service 509fd4
	// Create the final xmlrpc client with our custom RoundTripper handling
Packit Service 509fd4
	// sessionID, sessionKey and callnum
Packit Service 509fd4
	kojiTransport := &Transport{
Packit Service 509fd4
		sessionID:  reply.SessionID,
Packit Service 509fd4
		sessionKey: reply.SessionKey,
Packit Service 509fd4
		callnum:    0,
Packit Service 509fd4
		transport:  transport,
Packit Service 509fd4
	}
Packit Service 509fd4
	client, err := xmlrpc.NewClient(server, kojiTransport)
Packit Service 509fd4
	if err != nil {
Packit Service 509fd4
		return nil, err
Packit 63bb0d
	}
Packit 63bb0d
Packit Service 509fd4
	return &Koji{
Packit Service 509fd4
		xmlrpc:    client,
Packit Service 509fd4
		server:    server,
Packit Service 509fd4
		transport: kojiTransport,
Packit Service 509fd4
	}, nil
Packit Service 509fd4
}
Packit 63bb0d
Packit Service 509fd4
// NewFromPlain creates a new Koji sessions  =authenticated using the plain
Packit Service 509fd4
// username/password method. If you want to speak to a public koji instance,
Packit Service 509fd4
// you probably cannot use this method.
Packit Service 509fd4
func NewFromPlain(server, user, password string, transport http.RoundTripper) (*Koji, error) {
Packit Service 509fd4
	// Create a temporary xmlrpc client.
Packit Service 509fd4
	// The API doesn't require sessionID, sessionKey and callnum yet,
Packit Service 509fd4
	// so there's no need to use the custom Koji RoundTripper,
Packit Service 509fd4
	// let's just use the one that the called passed in.
Packit Service 509fd4
	loginClient, err := xmlrpc.NewClient(server, http.DefaultTransport)
Packit Service 509fd4
	if err != nil {
Packit Service 509fd4
		return nil, err
Packit Service 509fd4
	}
Packit 63bb0d
Packit Service 509fd4
	args := []interface{}{user, password}
Packit Service 509fd4
	var reply loginReply
Packit Service 509fd4
	err = loginClient.Call("login", args, &reply)
Packit Service 509fd4
	if err != nil {
Packit Service 509fd4
		return nil, err
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 509fd4
	return newKoji(server, transport, reply)
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
// NewFromGSSAPI creates a new Koji session authenticated using GSSAPI.
Packit Service 509fd4
// Principal and keytab used for the session is passed using credentials
Packit Service 509fd4
// parameter.
Packit Service 509fd4
func NewFromGSSAPI(server string, credentials *GSSAPICredentials, transport http.RoundTripper) (*Koji, error) {
Packit Service 509fd4
	// Create a temporary xmlrpc client with kerberos transport.
Packit Service 509fd4
	// The API doesn't require sessionID, sessionKey and callnum yet,
Packit Service 509fd4
	// so there's no need to use the custom Koji RoundTripper,
Packit Service 509fd4
	// let's just use the one that the called passed in.
Packit Service 509fd4
	loginClient, err := xmlrpc.NewClient(server+"/ssllogin", &khttp.Transport{
Packit Service 509fd4
		KeyTab:    credentials.KeyTab,
Packit Service 509fd4
		Principal: credentials.Principal,
Packit Service 509fd4
		Next:      transport,
Packit Service 509fd4
	})
Packit Service 509fd4
	if err != nil {
Packit Service 509fd4
		return nil, err
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 509fd4
	var reply loginReply
Packit Service 509fd4
	err = loginClient.Call("sslLogin", nil, &reply)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return nil, err
Packit 63bb0d
	}
Packit Service 509fd4
Packit Service 509fd4
	return newKoji(server, transport, reply)
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// GetAPIVersion gets the version of the API of the remote Koji instance
Packit 63bb0d
func (k *Koji) GetAPIVersion() (int, error) {
Packit 63bb0d
	var version int
Packit 63bb0d
	err := k.xmlrpc.Call("getAPIVersion", nil, &version)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return 0, err
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return version, nil
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
// Logout ends the session
Packit Service 509fd4
func (k *Koji) Logout() error {
Packit Service 509fd4
	err := k.xmlrpc.Call("logout", nil, nil)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return err
Packit 63bb0d
	}
Packit 63bb0d
	return nil
Packit 63bb0d
}
Packit 63bb0d
Packit Service 509fd4
// CGInitBuild reserves a build ID and initializes a build
Packit Service 509fd4
func (k *Koji) CGInitBuild(name, version, release string) (*CGInitBuildResult, error) {
Packit Service 509fd4
	var buildInfo struct {
Packit Service 509fd4
		Name    string `xmlrpc:"name"`
Packit Service 509fd4
		Version string `xmlrpc:"version"`
Packit Service 509fd4
		Release string `xmlrpc:"release"`
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 509fd4
	buildInfo.Name = name
Packit Service 509fd4
	buildInfo.Version = version
Packit Service 509fd4
	buildInfo.Release = release
Packit Service 509fd4
Packit Service 509fd4
	var result CGInitBuildResult
Packit Service 509fd4
	err := k.xmlrpc.Call("CGInitBuild", []interface{}{"osbuild", buildInfo}, &result)
Packit 63bb0d
	if err != nil {
Packit Service 509fd4
		return nil, err
Packit 63bb0d
	}
Packit Service 509fd4
Packit Service 509fd4
	return &result, nil
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
/* from `koji/__init__.py`
Packit Service 509fd4
BUILD_STATES = Enum((
Packit Service 509fd4
    'BUILDING',
Packit Service 509fd4
    'COMPLETE',
Packit Service 509fd4
    'DELETED',
Packit Service 509fd4
    'FAILED',
Packit Service 509fd4
    'CANCELED',
Packit Service 509fd4
))
Packit Service 509fd4
*/
Packit Service 509fd4
const (
Packit Service 509fd4
	_ = iota /* BUILDING */
Packit Service 509fd4
	_        /* COMPLETED */
Packit Service 509fd4
	_        /* DELETED */
Packit Service 509fd4
	buildStateFailed
Packit Service 509fd4
	buildStateCanceled
Packit Service 509fd4
)
Packit Service 509fd4
Packit Service 509fd4
// CGFailBuild marks an in-progress build as failed
Packit Service 509fd4
func (k *Koji) CGFailBuild(buildID int, token string) error {
Packit Service 509fd4
	return k.xmlrpc.Call("CGRefundBuild", []interface{}{"osbuild", buildID, token, buildStateFailed}, nil)
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
// CGCancelBuild marks an in-progress build as cancelled, and
Packit Service 509fd4
func (k *Koji) CGCancelBuild(buildID int, token string) error {
Packit Service 509fd4
	return k.xmlrpc.Call("CGRefundBuild", []interface{}{"osbuild", buildID, token, buildStateCanceled}, nil)
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// CGImport imports previously uploaded content, by specifying its metadata, and the temporary
Packit 63bb0d
// directory where it is located.
Packit Service 509fd4
func (k *Koji) CGImport(build ImageBuild, buildRoots []BuildRoot, images []Image, directory, token string) (*CGImportResult, error) {
Packit 63bb0d
	m := &Metadata{
Packit Service 509fd4
		ImageBuild: build,
Packit 63bb0d
		BuildRoots: buildRoots,
Packit Service 509fd4
		Images:     images,
Packit 63bb0d
	}
Packit 63bb0d
	metadata, err := json.Marshal(m)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return nil, err
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	var result CGImportResult
Packit Service 509fd4
	err = k.xmlrpc.Call("CGImport", []interface{}{string(metadata), directory, token}, &result)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return nil, err
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return &result, nil
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// uploadChunk uploads a byte slice to a given filepath/filname at a given offset
Packit 63bb0d
func (k *Koji) uploadChunk(chunk []byte, filepath, filename string, offset uint64) error {
Packit 63bb0d
	// We have to open-code a bastardized version of XML-RPC: We send an octet-stream, as
Packit 63bb0d
	// if it was an RPC call, and get a regular XML-RPC reply back. In addition to the
Packit 63bb0d
	// standard URL parameters, we also have to pass any other parameters as part of the
Packit 63bb0d
	// URL, as the body can only contain the payload.
Packit 63bb0d
	u, err := url.Parse(k.server)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return err
Packit 63bb0d
	}
Packit 63bb0d
	q := u.Query()
Packit 63bb0d
	q.Add("filepath", filepath)
Packit 63bb0d
	q.Add("filename", filename)
Packit 63bb0d
	q.Add("offset", fmt.Sprintf("%v", offset))
Packit 63bb0d
	q.Add("fileverify", "adler32")
Packit 63bb0d
	u.RawQuery = q.Encode()
Packit 63bb0d
Packit Service 509fd4
	client := http.Client{Transport: k.transport}
Packit Service 509fd4
	respData, err := client.Post(u.String(), "application/octet-stream", bytes.NewBuffer(chunk))
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return err
Packit 63bb0d
	}
Packit Service 509fd4
	defer respData.Body.Close()
Packit 63bb0d
Packit Service 509fd4
	body, err := ioutil.ReadAll(respData.Body)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return err
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	var reply struct {
Packit 63bb0d
		Size      int    `xmlrpc:"size"`
Packit 63bb0d
		HexDigest string `xmlrpc:"hexdigest"`
Packit 63bb0d
	}
Packit 63bb0d
Packit Service 509fd4
	err = processXMLRPCResponse(body, &reply)
Packit 63bb0d
	if err != nil {
Packit Service 509fd4
		return err
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	if reply.Size != len(chunk) {
Packit 63bb0d
		return fmt.Errorf("Sent a chunk of %d bytes, but server got %d bytes", len(chunk), reply.Size)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	digest := fmt.Sprintf("%08x", adler32.Checksum(chunk))
Packit 63bb0d
	if reply.HexDigest != digest {
Packit 63bb0d
		return fmt.Errorf("Sent a chunk with Adler32 digest %s, but server computed digest %s", digest, reply.HexDigest)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return nil
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
// Upload uploads file to the temporary filepath on the kojiserver under the name filename
Packit 63bb0d
// The md5sum and size of the file is returned on success.
Packit 63bb0d
func (k *Koji) Upload(file io.Reader, filepath, filename string) (string, uint64, error) {
Packit 63bb0d
	chunk := make([]byte, 1024*1024) // upload a megabyte at a time
Packit 63bb0d
	offset := uint64(0)
Packit 63bb0d
	hash := md5.New()
Packit 63bb0d
	for {
Packit 63bb0d
		n, err := file.Read(chunk)
Packit 63bb0d
		if err != nil {
Packit 63bb0d
			if err == io.EOF {
Packit 63bb0d
				break
Packit 63bb0d
			}
Packit 63bb0d
			return "", 0, err
Packit 63bb0d
		}
Packit 63bb0d
		err = k.uploadChunk(chunk[:n], filepath, filename, offset)
Packit 63bb0d
		if err != nil {
Packit 63bb0d
			return "", 0, err
Packit 63bb0d
		}
Packit 63bb0d
		offset += uint64(n)
Packit 63bb0d
Packit 63bb0d
		m, err := hash.Write(chunk[:n])
Packit 63bb0d
		if err != nil {
Packit 63bb0d
			return "", 0, err
Packit 63bb0d
		}
Packit 63bb0d
		if m != n {
Packit 63bb0d
			return "", 0, fmt.Errorf("sent %d bytes, but hashed %d", n, m)
Packit 63bb0d
		}
Packit 63bb0d
	}
Packit 63bb0d
	return fmt.Sprintf("%x", hash.Sum(nil)), offset, nil
Packit 63bb0d
}
Packit Service 509fd4
Packit Service 509fd4
type Transport struct {
Packit Service 509fd4
	sessionID  int64
Packit Service 509fd4
	sessionKey string
Packit Service 509fd4
	callnum    int
Packit Service 509fd4
	transport  http.RoundTripper
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
// RoundTrip implements the RoundTripper interface, using the default
Packit Service 509fd4
// transport. When a session has been established, also pass along the
Packit Service 509fd4
// session credentials. This may not be how the RoundTripper interface
Packit Service 509fd4
// is meant to be used, but the underlying XML-RPC helpers don't allow
Packit Service 509fd4
// us to adjust the URL per-call (these arguments should really be in
Packit Service 509fd4
// the body).
Packit Service 509fd4
func (rt *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
Packit Service 509fd4
	// Clone the request, so as not to alter the passed in value.
Packit Service 509fd4
	rClone := new(http.Request)
Packit Service 509fd4
	*rClone = *req
Packit Service 509fd4
	rClone.Header = make(http.Header, len(req.Header))
Packit Service 509fd4
	for idx, header := range req.Header {
Packit Service 509fd4
		rClone.Header[idx] = append([]string(nil), header...)
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 509fd4
	values := rClone.URL.Query()
Packit Service 509fd4
	values.Add("session-id", fmt.Sprintf("%v", rt.sessionID))
Packit Service 509fd4
	values.Add("session-key", rt.sessionKey)
Packit Service 509fd4
	values.Add("callnum", fmt.Sprintf("%v", rt.callnum))
Packit Service 509fd4
	rClone.URL.RawQuery = values.Encode()
Packit Service 509fd4
Packit Service 509fd4
	// Each call is given a unique callnum.
Packit Service 509fd4
	rt.callnum++
Packit Service 509fd4
Packit Service 509fd4
	return rt.transport.RoundTrip(rClone)
Packit Service 509fd4
}
Packit Service 509fd4
Packit Service 509fd4
func GSSAPICredentialsFromEnv() (*GSSAPICredentials, error) {
Packit Service 509fd4
	principal, principalExists := os.LookupEnv("OSBUILD_COMPOSER_KOJI_PRINCIPAL")
Packit Service 509fd4
	keyTab, keyTabExists := os.LookupEnv("OSBUILD_COMPOSER_KOJI_KEYTAB")
Packit Service 509fd4
Packit Service 509fd4
	if !principalExists || !keyTabExists {
Packit Service 509fd4
		return nil, errors.New("Both OSBUILD_COMPOSER_KOJI_PRINCIPAL and OSBUILD_COMPOSER_KOJI_KEYTAB must be set")
Packit Service 509fd4
	}
Packit Service 509fd4
Packit Service 509fd4
	return &GSSAPICredentials{
Packit Service 509fd4
		Principal: principal,
Packit Service 509fd4
		KeyTab:    keyTab,
Packit Service 509fd4
	}, nil
Packit Service 509fd4
}