Skip to content

Commit e408963

Browse files
jakobmoellerdevjakobmoellerdev
authored
feat: support index based reading of OCI artifacts (#1646)
<!-- markdownlint-disable MD041 --> #### What this PR does / why we need it Refactored `ComponentVersionContainer` to allow fetching OCI manifests from top level OCI Image indexes. This is preparation for allowing the reference of native OCI Manifests that are themselves referenced via index first. This is prep work to allow native storage in OCI registries for local blobs that might themselves be stored previously as OCM Artifact Sets. They could now be accessed natively instead. #### Which issue(s) this PR is related to <!-- Usage: `Related to #<issue number>`, or `Related to (paste link of issue)`. --> This allows native reading of OCM Artifacts created with the new v2 library that can use indexes to store artifacts natively final stepping stone to prepare ADR for open-component-model/ocm-project#680 fixes open-component-model/ocm-project#717 --------- Signed-off-by: Jakob Möller <[email protected]> Signed-off-by: Jakob Möller <[email protected]>
1 parent 7a2ada0 commit e408963

File tree

14 files changed

+438
-65
lines changed

14 files changed

+438
-65
lines changed

‎api/oci/extensions/repositories/artifactset/ctf_roundtrip_test.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -205,10 +205,10 @@ var _ = Describe("CTF to CTF-with-resource to OCI roundtrip", Ordered, func() {
205205
Expect(config.Config.Entrypoint).ToNot(BeEmpty())
206206
})
207207

208-
It("copies OCI archive to Docker with skopeo", func() {
208+
It("copies OCI archive to Docker with skopeo", func(ctx SpecContext) {
209209
// Use skopeo to copy from OCI archive (tgz) to docker daemon
210210
imageTag = "ocm-test-hello:" + resourceVersion
211-
cmd := exec.Command("skopeo", "copy",
211+
cmd := exec.CommandContext(ctx, "skopeo", "copy",
212212
"oci-archive:"+resourcesOciTgz+":"+resourceVersion,
213213
"docker-daemon:"+imageTag,
214214
"--override-os=linux")
@@ -225,8 +225,7 @@ var _ = Describe("CTF to CTF-with-resource to OCI roundtrip", Ordered, func() {
225225
Expect(string(out)).To(ContainSubstring("Hello OCM!"))
226226
})
227227

228-
It("downloads resource from target CTF without --oci-layout and verifies it", func() {
229-
ctx := context.Background()
228+
It("downloads resource from target CTF without --oci-layout and verifies it", func(ctx SpecContext) {
230229
resourcesOcmTgz = filepath.Join(tempDir, "resource-ocm-layout")
231230

232231
buf := bytes.NewBuffer(nil)

‎api/oci/extensions/repositories/artifactset/format.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -244,12 +244,13 @@ func OpenFromDataAccess(acc accessobj.AccessMode, mediatype string, data blobacc
244244
}
245245
defer reader.Close()
246246
o.SetReader(reader)
247-
fmt := accessio.FormatTar
248-
249-
if mime.IsGZip(mediatype) {
250-
fmt = accessio.FormatTGZ
247+
if o.GetFileFormat() == nil {
248+
fmt := accessio.FormatTar
249+
if mime.IsGZip(mediatype) {
250+
fmt = accessio.FormatTGZ
251+
}
252+
o.SetFileFormat(fmt)
251253
}
252-
o.SetFileFormat(fmt)
253254
return Open(acc&accessobj.ACC_READONLY, "", 0, o)
254255
}
255256

‎api/oci/extensions/repositories/artifactset/utils_synthesis.go‎

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -84,35 +84,32 @@ func SynthesizeArtifactBlobWithFilter(ns cpi.NamespaceAccess, ref string, filter
8484
return nil, err
8585
}
8686
}
87-
return SynthesizeArtifactBlobForArtifact(art, ref, filter)
87+
return SynthesizeArtifactBlobForArtifact(art, []string{ref}, filter)
8888
}
8989

90-
func SynthesizeArtifactBlobForArtifact(art cpi.ArtifactAccess, ref string, filter ...filters.Filter) (ArtifactBlob, error) {
90+
func SynthesizeArtifactBlobForArtifact(art cpi.ArtifactAccess, refs []string, filter ...filters.Filter) (ArtifactBlob, error) {
9191
blob, err := art.Blob()
9292
if err != nil {
9393
return nil, err
9494
}
9595

96-
vers, err := ociutils.ParseVersion(ref)
97-
if err != nil {
98-
return nil, err
96+
tags := make([]string, 0, len(refs))
97+
for _, ref := range refs {
98+
vers, err := ociutils.ParseVersion(ref)
99+
if err != nil {
100+
return nil, err
101+
}
102+
if vers.IsTagged() {
103+
tags = append(tags, vers.GetTag())
104+
}
99105
}
100106

101107
return SythesizeArtifactSet(func(set *ArtifactSet) (string, error) {
102-
dig, err := transfer.TransferArtifactWithFilter(art, set, filters.And(filter...))
108+
dig, err := transfer.TransferArtifactWithFilter(art, set, filters.And(filter...), tags...)
103109
if err != nil {
104110
return "", fmt.Errorf("failed to transfer artifact: %w", err)
105111
}
106-
107-
if ok := vers.IsTagged(); ok {
108-
err = set.AddTags(*dig, vers.GetTag())
109-
if err != nil {
110-
return "", fmt.Errorf("failed to add tag: %w", err)
111-
}
112-
}
113-
114112
set.Annotate(MAINARTIFACT_ANNOTATION, dig.String())
115-
116113
return blob.MimeType(), nil
117114
})
118115
}

‎api/oci/extensions/repositories/ctf/index/ctfindex.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ type ArtifactMeta struct {
1818
Repository string `json:"repository"`
1919
Tag string `json:"tag,omitempty"`
2020
Digest digest.Digest `json:"digest,omitempty"`
21+
MediaType string `json:"mediaType,omitempty"`
2122
}
2223

2324
func Decode(data []byte) (*ArtifactIndex, error) {

‎api/oci/extensions/repositories/ctf/index/index.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,7 @@ func (r *RepositoryIndex) GetDescriptor() *ArtifactIndex {
241241
Repository: vers.Repository,
242242
Tag: vers.Tag,
243243
Digest: vers.Digest,
244+
MediaType: vers.MediaType,
244245
}
245246
index.Index = append(index.Index, *d)
246247
}

‎api/oci/extensions/repositories/ctf/namespace.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,6 @@ func (n *namespaceContainer) Close() error {
4040
return nil
4141
}
4242

43-
func (n *namespaceContainer) GetBlobDescriptor(digest digest.Digest) *cpi.Descriptor {
44-
return nil
45-
}
46-
4743
func (n *namespaceContainer) ListTags() ([]string, error) {
4844
return n.repo.getIndex().GetTags(n.impl.GetNamespace()), nil // return digests as tags, also
4945
}
@@ -84,6 +80,7 @@ func (n *namespaceContainer) AddArtifact(artifact cpi.Artifact, tags ...string)
8480
Repository: n.impl.GetNamespace(),
8581
Tag: "",
8682
Digest: blob.Digest(),
83+
MediaType: blob.MimeType(),
8784
})
8885
return blob, n.AddTags(blob.Digest(), tags...)
8986
}

‎api/ocm/extensions/accessmethods/ociartifact/method.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ const (
3535
const (
3636
LegacyType = "ociRegistry"
3737
LegacyTypeV1 = LegacyType + runtime.VersionSeparator + "v1"
38+
39+
LegacyType2 = "OCIImage"
40+
LegacyType2V1 = LegacyType2 + runtime.VersionSeparator + "v1"
3841
)
3942

4043
func init() {
@@ -43,6 +46,9 @@ func init() {
4346

4447
accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyType))
4548
accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyTypeV1))
49+
50+
accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyType2))
51+
accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyType2V1))
4652
}
4753

4854
func Is(spec accspeccpi.AccessSpec) bool {
@@ -355,7 +361,7 @@ func (m *accessMethod) getBlob() (artifactset.ArtifactBlob, error) {
355361
}
356362
logger := Logger(WrapContextProvider(m.ctx))
357363
logger.Info("synthesize artifact blob", "ref", m.reference)
358-
m.blob, err = artifactset.SynthesizeArtifactBlobForArtifact(m.art, m.ref.VersionSpec())
364+
m.blob, err = artifactset.SynthesizeArtifactBlobForArtifact(m.art, []string{m.ref.VersionSpec()})
359365
logger.Info("synthesize artifact blob done", "ref", m.reference, "error", logging.ErrorMessage(err))
360366
if err != nil {
361367
m.err = err

‎api/ocm/extensions/repositories/genericocireg/accessmethod_localblob.go‎

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package genericocireg
22

33
import (
44
"bytes"
5+
"fmt"
56
"io"
67
"os"
78
"strings"
@@ -13,6 +14,7 @@ import (
1314

1415
"ocm.software/ocm/api/oci"
1516
"ocm.software/ocm/api/oci/artdesc"
17+
"ocm.software/ocm/api/oci/extensions/repositories/artifactset"
1618
"ocm.software/ocm/api/ocm/cpi/accspeccpi"
1719
"ocm.software/ocm/api/ocm/extensions/accessmethods/localblob"
1820
"ocm.software/ocm/api/utils/blobaccess/blobaccess"
@@ -26,6 +28,7 @@ type localBlobAccessMethod struct {
2628
spec *localblob.AccessSpec
2729
namespace oci.NamespaceAccess
2830
artifact oci.ArtifactAccess
31+
mimeType string
2932
}
3033

3134
var _ accspeccpi.AccessMethodImpl = (*localBlobAccessMethod)(nil)
@@ -40,6 +43,11 @@ func newLocalBlobAccessMethodImpl(a *localblob.AccessSpec, ns oci.NamespaceAcces
4043
namespace: ns,
4144
artifact: art,
4245
}
46+
if m.spec.MediaType == artdesc.MediaTypeImageIndex || m.spec.MediaType == artdesc.MediaTypeImageManifest {
47+
// if we discover a localblob with an index or manifest media type, we can
48+
// assume that we are dealing with a new style of artifact created by the new reference library.
49+
m.mimeType = artifactset.MediaType(m.spec.MediaType)
50+
}
4351
ref.BeforeCleanup(refmgmt.CleanupHandlerFunc(m.cache))
4452
return m, nil
4553
}
@@ -99,8 +107,32 @@ func (m *localBlobAccessMethod) getBlob() (blobaccess.DataAccess, error) {
99107
err error
100108
)
101109
if len(refs) < 2 {
102-
_, data, err = m.namespace.GetBlobData(digest.Digest(m.spec.LocalReference))
103-
if err != nil {
110+
if m.spec.MediaType == artdesc.MediaTypeImageIndex || m.spec.MediaType == artdesc.MediaTypeImageManifest {
111+
// if we have a nested manifest or index, we can use the blob synthesis utility here to download
112+
// the entire artifact set.
113+
art, err := m.namespace.GetArtifact(m.spec.LocalReference)
114+
if err != nil {
115+
return nil, fmt.Errorf("failed to get artifact for local reference %q: %w", m.spec.LocalReference, err)
116+
}
117+
defer art.Close()
118+
var artifactRefs []string
119+
if m.spec.ReferenceName != "" {
120+
// if we have a reference name, it consists of repository and tag
121+
// so we can extract the tag to use it
122+
refSpec, err := oci.ParseRef(m.spec.ReferenceName)
123+
if err != nil {
124+
return nil, fmt.Errorf("failed to parse reference name %q: %w", m.spec.ReferenceName, err)
125+
}
126+
if refSpec.GetTag() != "" {
127+
artifactRefs = append(artifactRefs, refSpec.GetTag())
128+
}
129+
}
130+
artblob, err := artifactset.SynthesizeArtifactBlobForArtifact(art, artifactRefs)
131+
if err != nil {
132+
return nil, fmt.Errorf("failed to synthesize artifact blob: %w", err)
133+
}
134+
data = artblob
135+
} else if _, data, err = m.namespace.GetBlobData(digest.Digest(m.spec.LocalReference)); err != nil {
104136
return nil, err
105137
}
106138
} else {
@@ -123,10 +155,13 @@ func (m *localBlobAccessMethod) Get() ([]byte, error) {
123155
}
124156

125157
func (m *localBlobAccessMethod) MimeType() string {
158+
if m.mimeType != "" {
159+
return m.mimeType
160+
}
126161
return m.spec.MediaType
127162
}
128163

129-
////////////////////////////////////////////////////////////////////////////////
164+
// //////////////////////////////////////////////////////////////////////////////
130165

131166
type composedBlock struct {
132167
m *localBlobAccessMethod

‎api/ocm/extensions/repositories/genericocireg/componentversion.go‎

Lines changed: 75 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -46,38 +46,82 @@ func newComponentVersionAccess(mode accessobj.AccessMode, comp *componentAccessI
4646
return &repocpi.ComponentVersionAccessInfo{c, true, persistent}, nil
4747
}
4848

49-
////////////////////////////////////////////////////////////////////////////////
49+
// //////////////////////////////////////////////////////////////////////////////
5050

5151
type ComponentVersionContainer struct {
5252
bridge repocpi.ComponentVersionAccessBridge
5353

54-
comp *componentAccessImpl
55-
version string
56-
access oci.ArtifactAccess
57-
manifest oci.ManifestAccess
58-
state accessobj.State
54+
comp *componentAccessImpl
55+
version string
56+
indexArtifact oci.ArtifactAccess
57+
manifestArtifact oci.ArtifactAccess
58+
manifest oci.ManifestAccess
59+
index oci.IndexAccess
60+
state accessobj.State
5961
}
6062

6163
var _ repocpi.ComponentVersionAccessImpl = (*ComponentVersionContainer)(nil)
6264

6365
func newComponentVersionContainer(mode accessobj.AccessMode, comp *componentAccessImpl, version string, access oci.ArtifactAccess) (*ComponentVersionContainer, error) {
64-
m := access.ManifestAccess()
65-
if m == nil {
66-
return nil, errors.ErrInvalid("artifact type")
66+
var m oci.ManifestAccess
67+
var i oci.IndexAccess
68+
69+
var manifestArtifact, indexArtifact oci.ArtifactAccess
70+
71+
if access.IsIndex() {
72+
idx, err := access.Index()
73+
if err != nil {
74+
return nil, err
75+
}
76+
if len(idx.Manifests) < 1 {
77+
return nil, fmt.Errorf("index has no manifests")
78+
}
79+
first := idx.Manifests[0]
80+
firstArtifact, err := access.GetArtifact(first.Digest)
81+
if err != nil {
82+
return nil, err
83+
}
84+
if !firstArtifact.IsManifest() {
85+
return nil, fmt.Errorf("first manifest in index is not a manifest")
86+
}
87+
88+
m = firstArtifact.ManifestAccess()
89+
manifestArtifact = firstArtifact
90+
i = access.IndexAccess()
91+
indexArtifact = access
92+
} else {
93+
m = access.ManifestAccess()
94+
if m == nil {
95+
return nil, fmt.Errorf("artifact is neither manifest nor index")
96+
}
97+
98+
manifestArtifact = access
6799
}
68-
state, err := NewState(mode, comp.name, version, m, compatattr.Get(comp.GetContext()))
100+
101+
state, err := NewState(mode, comp.name, version, i, m, compatattr.Get(comp.GetContext()))
69102
if err != nil {
70-
access.Close()
103+
err = errors.Join(err, manifestArtifact.Close())
104+
if indexArtifact != nil {
105+
err = errors.Join(err, indexArtifact.Close())
106+
}
71107
return nil, err
72108
}
73109

74-
return &ComponentVersionContainer{
75-
comp: comp,
76-
version: version,
77-
access: access,
78-
manifest: m,
79-
state: state,
80-
}, nil
110+
container := &ComponentVersionContainer{
111+
comp: comp,
112+
version: version,
113+
manifestArtifact: manifestArtifact,
114+
indexArtifact: indexArtifact,
115+
manifest: m,
116+
state: state,
117+
}
118+
119+
if indexArtifact != nil {
120+
// index based manifests are optional and only read based for next gen support
121+
container.SetReadOnly()
122+
}
123+
124+
return container, nil
81125
}
82126

83127
func (c *ComponentVersionContainer) SetBridge(impl repocpi.ComponentVersionAccessBridge) {
@@ -89,11 +133,20 @@ func (c *ComponentVersionContainer) GetParentBridge() repocpi.ComponentAccessBri
89133
}
90134

91135
func (c *ComponentVersionContainer) Close() error {
92-
if c.manifest == nil {
136+
if c.manifest == nil && c.manifestArtifact == nil {
93137
return accessio.ErrClosed
94138
}
95139
c.manifest = nil
96-
return c.access.Close()
140+
c.index = nil
141+
142+
var err error
143+
if c.indexArtifact != nil {
144+
err = errors.Join(err, c.indexArtifact.Close())
145+
}
146+
if c.manifestArtifact != nil {
147+
err = errors.Join(err, c.manifestArtifact.Close())
148+
}
149+
return err
97150
}
98151

99152
func (c *ComponentVersionContainer) SetReadOnly() {
@@ -140,9 +193,9 @@ func (c *ComponentVersionContainer) AccessMethod(a cpi.AccessSpec, cv refmgmt.Ex
140193

141194
switch a.GetKind() {
142195
case localblob.Type:
143-
return newLocalBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c.comp.namespace, c.access, cv)
196+
return newLocalBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c.comp.namespace, c.manifestArtifact, cv)
144197
case localociblob.Type:
145-
return newLocalOCIBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c.comp.namespace, c.access, cv)
198+
return newLocalOCIBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c.comp.namespace, c.manifestArtifact, cv)
146199
case relativeociref.Type:
147200
m, err := ociartifact.NewMethod(c.GetContext(), a, accessSpec.(*relativeociref.AccessSpec).Reference, c.comp.repo.ocirepo)
148201
if err == nil {

‎api/ocm/extensions/repositories/genericocireg/info.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func (h handler) Info(m cpi.ManifestAccess, config []byte) interface{} {
2828
info := &ComponentVersionInfo{
2929
Description: "component version",
3030
}
31-
acc := NewStateAccess(m)
31+
acc := NewStateAccess(nil, m)
3232
data, err := blobaccess.BlobData(acc.Get())
3333
if err != nil {
3434
info.Error = "cannot read component descriptor: " + err.Error()
@@ -46,7 +46,7 @@ func (h handler) Info(m cpi.ManifestAccess, config []byte) interface{} {
4646

4747
func (h handler) Description(pr common.Printer, m cpi.ManifestAccess, config []byte) {
4848
pr.Printf("component version:\n")
49-
acc := NewStateAccess(m)
49+
acc := NewStateAccess(nil, m)
5050
data, err := blobaccess.BlobData(acc.Get())
5151
if err != nil {
5252
pr.Printf(" cannot read component descriptor: %s\n", err.Error())

0 commit comments

Comments
 (0)