Blame internal/store/json.go

Packit 63bb0d
package store
Packit 63bb0d
Packit 63bb0d
import (
Packit 63bb0d
	"errors"
Packit 63bb0d
	"log"
Packit 63bb0d
	"sort"
Packit 63bb0d
	"time"
Packit 63bb0d
Packit 63bb0d
	"github.com/google/uuid"
Packit 63bb0d
	"github.com/osbuild/osbuild-composer/internal/blueprint"
Packit 63bb0d
	"github.com/osbuild/osbuild-composer/internal/common"
Packit 63bb0d
	"github.com/osbuild/osbuild-composer/internal/distro"
Packit 63bb0d
	"github.com/osbuild/osbuild-composer/internal/target"
Packit 63bb0d
)
Packit 63bb0d
Packit 63bb0d
type storeV0 struct {
Packit 63bb0d
	Blueprints blueprintsV0 `json:"blueprints"`
Packit 63bb0d
	Workspace  workspaceV0  `json:"workspace"`
Packit 63bb0d
	Composes   composesV0   `json:"composes"`
Packit 63bb0d
	Sources    sourcesV0    `json:"sources"`
Packit 63bb0d
	Changes    changesV0    `json:"changes"`
Packit 63bb0d
	Commits    commitsV0    `json:"commits"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type blueprintsV0 map[string]blueprint.Blueprint
Packit 63bb0d
type workspaceV0 map[string]blueprint.Blueprint
Packit 63bb0d
Packit 63bb0d
// A Compose represent the task of building a set of images from a single blueprint.
Packit 63bb0d
// It contains all the information necessary to generate the inputs for the job, as
Packit 63bb0d
// well as the job's state.
Packit 63bb0d
type composeV0 struct {
Packit 63bb0d
	Blueprint   *blueprint.Blueprint `json:"blueprint"`
Packit 63bb0d
	ImageBuilds []imageBuildV0       `json:"image_builds"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type composesV0 map[uuid.UUID]composeV0
Packit 63bb0d
Packit 63bb0d
// ImageBuild represents a single image build inside a compose
Packit 63bb0d
type imageBuildV0 struct {
Packit 63bb0d
	ID          int              `json:"id"`
Packit 63bb0d
	ImageType   string           `json:"image_type"`
Packit 63bb0d
	Manifest    distro.Manifest  `json:"manifest"`
Packit 63bb0d
	Targets     []*target.Target `json:"targets"`
Packit 63bb0d
	JobCreated  time.Time        `json:"job_created"`
Packit 63bb0d
	JobStarted  time.Time        `json:"job_started"`
Packit 63bb0d
	JobFinished time.Time        `json:"job_finished"`
Packit 63bb0d
	Size        uint64           `json:"size"`
Packit 63bb0d
	JobID       uuid.UUID        `json:"jobid,omitempty"`
Packit 63bb0d
Packit 63bb0d
	// Kept for backwards compatibility. Image builds which were done
Packit 63bb0d
	// before the move to the job queue use this to store whether they
Packit 63bb0d
	// finished successfully.
Packit 63bb0d
	QueueStatus common.ImageBuildState `json:"queue_status,omitempty"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type sourceV0 struct {
Packit 63bb0d
	Name     string `json:"name"`
Packit 63bb0d
	Type     string `json:"type"`
Packit 63bb0d
	URL      string `json:"url"`
Packit 63bb0d
	CheckGPG bool   `json:"check_gpg"`
Packit 63bb0d
	CheckSSL bool   `json:"check_ssl"`
Packit 63bb0d
	System   bool   `json:"system"`
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type sourcesV0 map[string]sourceV0
Packit 63bb0d
Packit 63bb0d
type changeV0 struct {
Packit 63bb0d
	Commit    string `json:"commit"`
Packit 63bb0d
	Message   string `json:"message"`
Packit 63bb0d
	Revision  *int   `json:"revision"`
Packit 63bb0d
	Timestamp string `json:"timestamp"`
Packit 63bb0d
	// BUG: We are currently not (un)marshalling the Blueprint field.
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
type changesV0 map[string]map[string]changeV0
Packit 63bb0d
Packit 63bb0d
type commitsV0 map[string][]string
Packit 63bb0d
Packit 63bb0d
func newBlueprintsFromV0(blueprintsStruct blueprintsV0) map[string]blueprint.Blueprint {
Packit 63bb0d
	blueprints := make(map[string]blueprint.Blueprint)
Packit 63bb0d
	for name, blueprint := range blueprintsStruct {
Packit 63bb0d
		blueprints[name] = blueprint.DeepCopy()
Packit 63bb0d
	}
Packit 63bb0d
	return blueprints
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newWorkspaceFromV0(workspaceStruct workspaceV0) map[string]blueprint.Blueprint {
Packit 63bb0d
	workspace := make(map[string]blueprint.Blueprint)
Packit 63bb0d
	for name, blueprint := range workspaceStruct {
Packit 63bb0d
		workspace[name] = blueprint.DeepCopy()
Packit 63bb0d
	}
Packit 63bb0d
	return workspace
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newComposesFromV0(composesStruct composesV0, arch distro.Arch, log *log.Logger) map[uuid.UUID]Compose {
Packit 63bb0d
	composes := make(map[uuid.UUID]Compose)
Packit 63bb0d
Packit 63bb0d
	for composeID, composeStruct := range composesStruct {
Packit 63bb0d
		c, err := newComposeFromV0(composeStruct, arch)
Packit 63bb0d
		if err != nil {
Packit 63bb0d
			if log != nil {
Packit 63bb0d
				log.Printf("ignoring compose: %v", err)
Packit 63bb0d
			}
Packit 63bb0d
			continue
Packit 63bb0d
		}
Packit 63bb0d
		composes[composeID] = c
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return composes
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newImageBuildFromV0(imageBuildStruct imageBuildV0, arch distro.Arch) (ImageBuild, error) {
Packit 63bb0d
	imgType := imageTypeFromCompatString(imageBuildStruct.ImageType, arch)
Packit 63bb0d
	if imgType == nil {
Packit 63bb0d
		// Invalid type strings in serialization format, this may happen
Packit 63bb0d
		// on upgrades.
Packit 63bb0d
		return ImageBuild{}, errors.New("invalid Image Type string")
Packit 63bb0d
	}
Packit 63bb0d
	// Backwards compatibility: fail all builds that are queued or
Packit 63bb0d
	// running. Jobs status is now handled outside of the store
Packit 63bb0d
	// (and the compose). The fields are kept so that previously
Packit 63bb0d
	// succeeded builds still show up correctly.
Packit 63bb0d
	queueStatus := imageBuildStruct.QueueStatus
Packit 63bb0d
	switch queueStatus {
Packit 63bb0d
	case common.IBRunning, common.IBWaiting:
Packit 63bb0d
		queueStatus = common.IBFailed
Packit 63bb0d
	}
Packit 63bb0d
	return ImageBuild{
Packit 63bb0d
		ID:          imageBuildStruct.ID,
Packit 63bb0d
		ImageType:   imgType,
Packit 63bb0d
		Manifest:    imageBuildStruct.Manifest,
Packit 63bb0d
		Targets:     imageBuildStruct.Targets,
Packit 63bb0d
		JobCreated:  imageBuildStruct.JobCreated,
Packit 63bb0d
		JobStarted:  imageBuildStruct.JobStarted,
Packit 63bb0d
		JobFinished: imageBuildStruct.JobFinished,
Packit 63bb0d
		Size:        imageBuildStruct.Size,
Packit 63bb0d
		JobID:       imageBuildStruct.JobID,
Packit 63bb0d
		QueueStatus: queueStatus,
Packit 63bb0d
	}, nil
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newComposeFromV0(composeStruct composeV0, arch distro.Arch) (Compose, error) {
Packit 63bb0d
	if len(composeStruct.ImageBuilds) != 1 {
Packit 63bb0d
		return Compose{}, errors.New("compose with unsupported number of image builds")
Packit 63bb0d
	}
Packit 63bb0d
	ib, err := newImageBuildFromV0(composeStruct.ImageBuilds[0], arch)
Packit 63bb0d
	if err != nil {
Packit 63bb0d
		return Compose{}, err
Packit 63bb0d
	}
Packit 63bb0d
	bp := composeStruct.Blueprint.DeepCopy()
Packit 63bb0d
	return Compose{
Packit 63bb0d
		Blueprint:  &bp,
Packit 63bb0d
		ImageBuild: ib,
Packit 63bb0d
	}, nil
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newSourceConfigsFromV0(sourcesStruct sourcesV0) map[string]SourceConfig {
Packit 63bb0d
	sources := make(map[string]SourceConfig)
Packit 63bb0d
Packit 63bb0d
	for name, source := range sourcesStruct {
Packit 63bb0d
		sources[name] = SourceConfig(source)
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return sources
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newChangesFromV0(changesStruct changesV0) map[string]map[string]blueprint.Change {
Packit 63bb0d
	changes := make(map[string]map[string]blueprint.Change)
Packit 63bb0d
Packit 63bb0d
	for name, commitsStruct := range changesStruct {
Packit 63bb0d
		commits := make(map[string]blueprint.Change)
Packit 63bb0d
		for commitID, change := range commitsStruct {
Packit 63bb0d
			commits[commitID] = blueprint.Change{
Packit 63bb0d
				Commit:    change.Commit,
Packit 63bb0d
				Message:   change.Message,
Packit 63bb0d
				Revision:  change.Revision,
Packit 63bb0d
				Timestamp: change.Timestamp,
Packit 63bb0d
			}
Packit 63bb0d
		}
Packit 63bb0d
		changes[name] = commits
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return changes
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newCommitsFromV0(commitsMapStruct commitsV0, changesMapStruct changesV0) map[string][]string {
Packit 63bb0d
	commitsMap := make(map[string][]string)
Packit 63bb0d
	for name, commitsStruct := range commitsMapStruct {
Packit 63bb0d
		commits := make([]string, len(commitsStruct))
Packit 63bb0d
		copy(commits, commitsStruct)
Packit 63bb0d
		commitsMap[name] = commits
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	// Populate BlueprintsCommits for existing blueprints without commit history
Packit 63bb0d
	// BlueprintsCommits tracks the order of the commits in BlueprintsChanges,
Packit 63bb0d
	// but may not be in-sync with BlueprintsChanges because it was added later.
Packit 63bb0d
	// This will sort the existing commits by timestamp and version to update
Packit 63bb0d
	// the store. BUT since the timestamp resolution is only 1s it is possible
Packit 63bb0d
	// that the order may be slightly wrong.
Packit 63bb0d
	for name, changes := range changesMapStruct {
Packit 63bb0d
		if _, exists := commitsMap[name]; !exists {
Packit 63bb0d
			changesSlice := make([]changeV0, 0, len(changes))
Packit 63bb0d
Packit 63bb0d
			// Copy the change objects from a map to a sortable slice
Packit 63bb0d
			for _, change := range changes {
Packit 63bb0d
				changesSlice = append(changesSlice, change)
Packit 63bb0d
			}
Packit 63bb0d
Packit 63bb0d
			// Sort the changes by Timestamp ascending
Packit 63bb0d
			sort.Slice(changesSlice, func(i, j int) bool {
Packit 63bb0d
				return changesSlice[i].Timestamp <= changesSlice[j].Timestamp
Packit 63bb0d
			})
Packit 63bb0d
Packit 63bb0d
			// Create a sorted list of commits based on the sorted list of change objects
Packit 63bb0d
			commits := make([]string, 0, len(changes))
Packit 63bb0d
			for _, c := range changesSlice {
Packit 63bb0d
				commits = append(commits, c.Commit)
Packit 63bb0d
			}
Packit 63bb0d
Packit 63bb0d
			// Assign the commits to the commit map, as an approximation of what we want
Packit 63bb0d
			commitsMap[name] = commits
Packit 63bb0d
		}
Packit 63bb0d
	}
Packit 63bb0d
Packit 63bb0d
	return commitsMap
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newStoreFromV0(storeStruct storeV0, arch distro.Arch, log *log.Logger) *Store {
Packit 63bb0d
	return &Store{
Packit 63bb0d
		blueprints:        newBlueprintsFromV0(storeStruct.Blueprints),
Packit 63bb0d
		workspace:         newWorkspaceFromV0(storeStruct.Workspace),
Packit 63bb0d
		composes:          newComposesFromV0(storeStruct.Composes, arch, log),
Packit 63bb0d
		sources:           newSourceConfigsFromV0(storeStruct.Sources),
Packit 63bb0d
		blueprintsChanges: newChangesFromV0(storeStruct.Changes),
Packit 63bb0d
		blueprintsCommits: newCommitsFromV0(storeStruct.Commits, storeStruct.Changes),
Packit 63bb0d
	}
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newBlueprintsV0(blueprints map[string]blueprint.Blueprint) blueprintsV0 {
Packit 63bb0d
	blueprintsStruct := make(blueprintsV0)
Packit 63bb0d
	for name, blueprint := range blueprints {
Packit 63bb0d
		blueprintsStruct[name] = blueprint.DeepCopy()
Packit 63bb0d
	}
Packit 63bb0d
	return blueprintsStruct
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newWorkspaceV0(workspace map[string]blueprint.Blueprint) workspaceV0 {
Packit 63bb0d
	workspaceStruct := make(workspaceV0)
Packit 63bb0d
	for name, blueprint := range workspace {
Packit 63bb0d
		workspaceStruct[name] = blueprint.DeepCopy()
Packit 63bb0d
	}
Packit 63bb0d
	return workspaceStruct
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newComposeV0(compose Compose) composeV0 {
Packit 63bb0d
	bp := compose.Blueprint.DeepCopy()
Packit 63bb0d
	return composeV0{
Packit 63bb0d
		Blueprint: &bp,
Packit 63bb0d
		ImageBuilds: []imageBuildV0{
Packit 63bb0d
			{
Packit 63bb0d
				ID:          compose.ImageBuild.ID,
Packit 63bb0d
				ImageType:   imageTypeToCompatString(compose.ImageBuild.ImageType),
Packit 63bb0d
				Manifest:    compose.ImageBuild.Manifest,
Packit 63bb0d
				Targets:     compose.ImageBuild.Targets,
Packit 63bb0d
				JobCreated:  compose.ImageBuild.JobCreated,
Packit 63bb0d
				JobStarted:  compose.ImageBuild.JobStarted,
Packit 63bb0d
				JobFinished: compose.ImageBuild.JobFinished,
Packit 63bb0d
				Size:        compose.ImageBuild.Size,
Packit 63bb0d
				JobID:       compose.ImageBuild.JobID,
Packit 63bb0d
				QueueStatus: compose.ImageBuild.QueueStatus,
Packit 63bb0d
			},
Packit 63bb0d
		},
Packit 63bb0d
	}
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newComposesV0(composes map[uuid.UUID]Compose) composesV0 {
Packit 63bb0d
	composesStruct := make(composesV0)
Packit 63bb0d
	for composeID, compose := range composes {
Packit 63bb0d
		composesStruct[composeID] = newComposeV0(compose)
Packit 63bb0d
	}
Packit 63bb0d
	return composesStruct
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newSourcesV0(sources map[string]SourceConfig) sourcesV0 {
Packit 63bb0d
	sourcesStruct := make(sourcesV0)
Packit 63bb0d
	for name, source := range sources {
Packit 63bb0d
		sourcesStruct[name] = sourceV0(source)
Packit 63bb0d
	}
Packit 63bb0d
	return sourcesStruct
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newChangesV0(changes map[string]map[string]blueprint.Change) changesV0 {
Packit 63bb0d
	changesStruct := make(changesV0)
Packit 63bb0d
	for name, commits := range changes {
Packit 63bb0d
		commitsStruct := make(map[string]changeV0)
Packit 63bb0d
		for commitID, change := range commits {
Packit 63bb0d
			commitsStruct[commitID] = changeV0{
Packit 63bb0d
				Commit:    change.Commit,
Packit 63bb0d
				Message:   change.Message,
Packit 63bb0d
				Revision:  change.Revision,
Packit 63bb0d
				Timestamp: change.Timestamp,
Packit 63bb0d
			}
Packit 63bb0d
		}
Packit 63bb0d
		changesStruct[name] = commitsStruct
Packit 63bb0d
	}
Packit 63bb0d
	return changesStruct
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func newCommitsV0(commits map[string][]string) commitsV0 {
Packit 63bb0d
	commitsStruct := make(commitsV0)
Packit 63bb0d
	for name, changes := range commits {
Packit 63bb0d
		commitsStruct[name] = changes
Packit 63bb0d
	}
Packit 63bb0d
	return commitsStruct
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func (store *Store) toStoreV0() *storeV0 {
Packit 63bb0d
	return &storeV0{
Packit 63bb0d
		Blueprints: newBlueprintsV0(store.blueprints),
Packit 63bb0d
		Workspace:  newWorkspaceV0(store.workspace),
Packit 63bb0d
		Composes:   newComposesV0(store.composes),
Packit 63bb0d
		Sources:    newSourcesV0(store.sources),
Packit 63bb0d
		Changes:    newChangesV0(store.blueprintsChanges),
Packit 63bb0d
		Commits:    newCommitsV0(store.blueprintsCommits),
Packit 63bb0d
	}
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
var imageTypeCompatMapping = map[string]string{
Packit Service 15f37d
	"vhd":                 "Azure",
Packit Service 15f37d
	"ami":                 "AWS",
Packit Service 15f37d
	"liveiso":             "LiveISO",
Packit Service 15f37d
	"openstack":           "OpenStack",
Packit Service 15f37d
	"qcow2":               "qcow2",
Packit Service 15f37d
	"vmdk":                "VMWare",
Packit Service 15f37d
	"ext4-filesystem":     "Raw-filesystem",
Packit Service 15f37d
	"partitioned-disk":    "Partitioned-disk",
Packit Service 15f37d
	"tar":                 "Tar",
Packit Service 15f37d
	"fedora-iot-commit":   "fedora-iot-commit",
Packit Service 15f37d
	"rhel-edge-commit":    "rhel-edge-commit",
Packit Service 15f37d
	"rhel-edge-container": "rhen-edge-container",
Packit Service 15f37d
	"rhel-edge-installer": "rhen-edge-installer",
Packit Service 15f37d
	"test_type":           "test_type",         // used only in json_test.go
Packit Service 15f37d
	"test_type_invalid":   "test_type_invalid", // used only in json_test.go
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func imageTypeToCompatString(imgType distro.ImageType) string {
Packit 63bb0d
	imgTypeString, exists := imageTypeCompatMapping[imgType.Name()]
Packit 63bb0d
	if !exists {
Packit 63bb0d
		panic("No mapping exists for " + imgType.Name())
Packit 63bb0d
	}
Packit 63bb0d
	return imgTypeString
Packit 63bb0d
}
Packit 63bb0d
Packit 63bb0d
func imageTypeFromCompatString(input string, arch distro.Arch) distro.ImageType {
Packit 63bb0d
	for k, v := range imageTypeCompatMapping {
Packit 63bb0d
		if v == input {
Packit 63bb0d
			imgType, err := arch.GetImageType(k)
Packit 63bb0d
			if err != nil {
Packit 63bb0d
				return nil
Packit 63bb0d
			}
Packit 63bb0d
			return imgType
Packit 63bb0d
		}
Packit 63bb0d
	}
Packit 63bb0d
	return nil
Packit 63bb0d
}