package store
import (
"errors"
"log"
"sort"
"time"
"github.com/google/uuid"
"github.com/osbuild/osbuild-composer/internal/blueprint"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/distro"
"github.com/osbuild/osbuild-composer/internal/target"
)
type storeV0 struct {
Blueprints blueprintsV0 `json:"blueprints"`
Workspace workspaceV0 `json:"workspace"`
Composes composesV0 `json:"composes"`
Sources sourcesV0 `json:"sources"`
Changes changesV0 `json:"changes"`
Commits commitsV0 `json:"commits"`
}
type blueprintsV0 map[string]blueprint.Blueprint
type workspaceV0 map[string]blueprint.Blueprint
// A Compose represent the task of building a set of images from a single blueprint.
// It contains all the information necessary to generate the inputs for the job, as
// well as the job's state.
type composeV0 struct {
Blueprint *blueprint.Blueprint `json:"blueprint"`
ImageBuilds []imageBuildV0 `json:"image_builds"`
}
type composesV0 map[uuid.UUID]composeV0
// ImageBuild represents a single image build inside a compose
type imageBuildV0 struct {
ID int `json:"id"`
ImageType string `json:"image_type"`
Manifest distro.Manifest `json:"manifest"`
Targets []*target.Target `json:"targets"`
JobCreated time.Time `json:"job_created"`
JobStarted time.Time `json:"job_started"`
JobFinished time.Time `json:"job_finished"`
Size uint64 `json:"size"`
JobID uuid.UUID `json:"jobid,omitempty"`
// Kept for backwards compatibility. Image builds which were done
// before the move to the job queue use this to store whether they
// finished successfully.
QueueStatus common.ImageBuildState `json:"queue_status,omitempty"`
}
type sourceV0 struct {
Name string `json:"name"`
Type string `json:"type"`
URL string `json:"url"`
CheckGPG bool `json:"check_gpg"`
CheckSSL bool `json:"check_ssl"`
System bool `json:"system"`
}
type sourcesV0 map[string]sourceV0
type changeV0 struct {
Commit string `json:"commit"`
Message string `json:"message"`
Revision *int `json:"revision"`
Timestamp string `json:"timestamp"`
// BUG: We are currently not (un)marshalling the Blueprint field.
}
type changesV0 map[string]map[string]changeV0
type commitsV0 map[string][]string
func newBlueprintsFromV0(blueprintsStruct blueprintsV0) map[string]blueprint.Blueprint {
blueprints := make(map[string]blueprint.Blueprint)
for name, blueprint := range blueprintsStruct {
blueprints[name] = blueprint.DeepCopy()
}
return blueprints
}
func newWorkspaceFromV0(workspaceStruct workspaceV0) map[string]blueprint.Blueprint {
workspace := make(map[string]blueprint.Blueprint)
for name, blueprint := range workspaceStruct {
workspace[name] = blueprint.DeepCopy()
}
return workspace
}
func newComposesFromV0(composesStruct composesV0, arch distro.Arch, log *log.Logger) map[uuid.UUID]Compose {
composes := make(map[uuid.UUID]Compose)
for composeID, composeStruct := range composesStruct {
c, err := newComposeFromV0(composeStruct, arch)
if err != nil {
if log != nil {
log.Printf("ignoring compose: %v", err)
}
continue
}
composes[composeID] = c
}
return composes
}
func newImageBuildFromV0(imageBuildStruct imageBuildV0, arch distro.Arch) (ImageBuild, error) {
imgType := imageTypeFromCompatString(imageBuildStruct.ImageType, arch)
if imgType == nil {
// Invalid type strings in serialization format, this may happen
// on upgrades.
return ImageBuild{}, errors.New("invalid Image Type string")
}
// Backwards compatibility: fail all builds that are queued or
// running. Jobs status is now handled outside of the store
// (and the compose). The fields are kept so that previously
// succeeded builds still show up correctly.
queueStatus := imageBuildStruct.QueueStatus
switch queueStatus {
case common.IBRunning, common.IBWaiting:
queueStatus = common.IBFailed
}
return ImageBuild{
ID: imageBuildStruct.ID,
ImageType: imgType,
Manifest: imageBuildStruct.Manifest,
Targets: imageBuildStruct.Targets,
JobCreated: imageBuildStruct.JobCreated,
JobStarted: imageBuildStruct.JobStarted,
JobFinished: imageBuildStruct.JobFinished,
Size: imageBuildStruct.Size,
JobID: imageBuildStruct.JobID,
QueueStatus: queueStatus,
}, nil
}
func newComposeFromV0(composeStruct composeV0, arch distro.Arch) (Compose, error) {
if len(composeStruct.ImageBuilds) != 1 {
return Compose{}, errors.New("compose with unsupported number of image builds")
}
ib, err := newImageBuildFromV0(composeStruct.ImageBuilds[0], arch)
if err != nil {
return Compose{}, err
}
bp := composeStruct.Blueprint.DeepCopy()
return Compose{
Blueprint: &bp,
ImageBuild: ib,
}, nil
}
func newSourceConfigsFromV0(sourcesStruct sourcesV0) map[string]SourceConfig {
sources := make(map[string]SourceConfig)
for name, source := range sourcesStruct {
sources[name] = SourceConfig(source)
}
return sources
}
func newChangesFromV0(changesStruct changesV0) map[string]map[string]blueprint.Change {
changes := make(map[string]map[string]blueprint.Change)
for name, commitsStruct := range changesStruct {
commits := make(map[string]blueprint.Change)
for commitID, change := range commitsStruct {
commits[commitID] = blueprint.Change{
Commit: change.Commit,
Message: change.Message,
Revision: change.Revision,
Timestamp: change.Timestamp,
}
}
changes[name] = commits
}
return changes
}
func newCommitsFromV0(commitsMapStruct commitsV0, changesMapStruct changesV0) map[string][]string {
commitsMap := make(map[string][]string)
for name, commitsStruct := range commitsMapStruct {
commits := make([]string, len(commitsStruct))
copy(commits, commitsStruct)
commitsMap[name] = commits
}
// Populate BlueprintsCommits for existing blueprints without commit history
// BlueprintsCommits tracks the order of the commits in BlueprintsChanges,
// but may not be in-sync with BlueprintsChanges because it was added later.
// This will sort the existing commits by timestamp and version to update
// the store. BUT since the timestamp resolution is only 1s it is possible
// that the order may be slightly wrong.
for name, changes := range changesMapStruct {
if _, exists := commitsMap[name]; !exists {
changesSlice := make([]changeV0, 0, len(changes))
// Copy the change objects from a map to a sortable slice
for _, change := range changes {
changesSlice = append(changesSlice, change)
}
// Sort the changes by Timestamp ascending
sort.Slice(changesSlice, func(i, j int) bool {
return changesSlice[i].Timestamp <= changesSlice[j].Timestamp
})
// Create a sorted list of commits based on the sorted list of change objects
commits := make([]string, 0, len(changes))
for _, c := range changesSlice {
commits = append(commits, c.Commit)
}
// Assign the commits to the commit map, as an approximation of what we want
commitsMap[name] = commits
}
}
return commitsMap
}
func newStoreFromV0(storeStruct storeV0, arch distro.Arch, log *log.Logger) *Store {
return &Store{
blueprints: newBlueprintsFromV0(storeStruct.Blueprints),
workspace: newWorkspaceFromV0(storeStruct.Workspace),
composes: newComposesFromV0(storeStruct.Composes, arch, log),
sources: newSourceConfigsFromV0(storeStruct.Sources),
blueprintsChanges: newChangesFromV0(storeStruct.Changes),
blueprintsCommits: newCommitsFromV0(storeStruct.Commits, storeStruct.Changes),
}
}
func newBlueprintsV0(blueprints map[string]blueprint.Blueprint) blueprintsV0 {
blueprintsStruct := make(blueprintsV0)
for name, blueprint := range blueprints {
blueprintsStruct[name] = blueprint.DeepCopy()
}
return blueprintsStruct
}
func newWorkspaceV0(workspace map[string]blueprint.Blueprint) workspaceV0 {
workspaceStruct := make(workspaceV0)
for name, blueprint := range workspace {
workspaceStruct[name] = blueprint.DeepCopy()
}
return workspaceStruct
}
func newComposeV0(compose Compose) composeV0 {
bp := compose.Blueprint.DeepCopy()
return composeV0{
Blueprint: &bp,
ImageBuilds: []imageBuildV0{
{
ID: compose.ImageBuild.ID,
ImageType: imageTypeToCompatString(compose.ImageBuild.ImageType),
Manifest: compose.ImageBuild.Manifest,
Targets: compose.ImageBuild.Targets,
JobCreated: compose.ImageBuild.JobCreated,
JobStarted: compose.ImageBuild.JobStarted,
JobFinished: compose.ImageBuild.JobFinished,
Size: compose.ImageBuild.Size,
JobID: compose.ImageBuild.JobID,
QueueStatus: compose.ImageBuild.QueueStatus,
},
},
}
}
func newComposesV0(composes map[uuid.UUID]Compose) composesV0 {
composesStruct := make(composesV0)
for composeID, compose := range composes {
composesStruct[composeID] = newComposeV0(compose)
}
return composesStruct
}
func newSourcesV0(sources map[string]SourceConfig) sourcesV0 {
sourcesStruct := make(sourcesV0)
for name, source := range sources {
sourcesStruct[name] = sourceV0(source)
}
return sourcesStruct
}
func newChangesV0(changes map[string]map[string]blueprint.Change) changesV0 {
changesStruct := make(changesV0)
for name, commits := range changes {
commitsStruct := make(map[string]changeV0)
for commitID, change := range commits {
commitsStruct[commitID] = changeV0{
Commit: change.Commit,
Message: change.Message,
Revision: change.Revision,
Timestamp: change.Timestamp,
}
}
changesStruct[name] = commitsStruct
}
return changesStruct
}
func newCommitsV0(commits map[string][]string) commitsV0 {
commitsStruct := make(commitsV0)
for name, changes := range commits {
commitsStruct[name] = changes
}
return commitsStruct
}
func (store *Store) toStoreV0() *storeV0 {
return &storeV0{
Blueprints: newBlueprintsV0(store.blueprints),
Workspace: newWorkspaceV0(store.workspace),
Composes: newComposesV0(store.composes),
Sources: newSourcesV0(store.sources),
Changes: newChangesV0(store.blueprintsChanges),
Commits: newCommitsV0(store.blueprintsCommits),
}
}
var imageTypeCompatMapping = map[string]string{
"vhd": "Azure",
"ami": "AWS",
"liveiso": "LiveISO",
"openstack": "OpenStack",
"qcow2": "qcow2",
"vmdk": "VMWare",
"ext4-filesystem": "Raw-filesystem",
"partitioned-disk": "Partitioned-disk",
"tar": "Tar",
"fedora-iot-commit": "fedora-iot-commit",
"rhel-edge-commit": "rhel-edge-commit",
"rhel-edge-container": "rhen-edge-container",
"rhel-edge-installer": "rhen-edge-installer",
"test_type": "test_type", // used only in json_test.go
"test_type_invalid": "test_type_invalid", // used only in json_test.go
}
func imageTypeToCompatString(imgType distro.ImageType) string {
imgTypeString, exists := imageTypeCompatMapping[imgType.Name()]
if !exists {
panic("No mapping exists for " + imgType.Name())
}
return imgTypeString
}
func imageTypeFromCompatString(input string, arch distro.Arch) distro.ImageType {
for k, v := range imageTypeCompatMapping {
if v == input {
imgType, err := arch.GetImageType(k)
if err != nil {
return nil
}
return imgType
}
}
return nil
}