diff --git a/build/build.go b/build/build.go
index 75373e64..fb312e4a 100644
--- a/build/build.go
+++ b/build/build.go
@@ -61,7 +61,6 @@ type Options struct {
 
 	Ref           string
 	Allow         []entitlements.Entitlement
-	Annotations   map[exptypes.AnnotationKey]string
 	Attests       map[string]*string
 	BuildArgs     map[string]string
 	CacheFrom     []client.CacheOptionsEntry
@@ -608,8 +607,12 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
 								}
 							}
 
-							filteredAnnotations := filterIndexAnnotations(opt.Annotations)
-							dt, desc, err := itpull.Combine(ctx, srcs, filteredAnnotations, false)
+							indexAnnotations, err := extractIndexAnnotations(opt.Exports)
+							if err != nil {
+								return err
+							}
+
+							dt, desc, err := itpull.Combine(ctx, srcs, indexAnnotations, false)
 							if err != nil {
 								return err
 							}
@@ -657,15 +660,25 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
 	return resp, nil
 }
 
-func filterIndexAnnotations(annotations map[exptypes.AnnotationKey]string) map[exptypes.AnnotationKey]string {
-	filteredAnnotations := map[exptypes.AnnotationKey]string{}
-	for k, v := range annotations {
-		switch k.Type {
-		case exptypes.AnnotationIndex, exptypes.AnnotationManifestDescriptor:
-			filteredAnnotations[k] = v
+func extractIndexAnnotations(exports []client.ExportEntry) (map[exptypes.AnnotationKey]string, error) {
+	annotations := map[exptypes.AnnotationKey]string{}
+	for _, exp := range exports {
+		for k, v := range exp.Attrs {
+			ak, ok, err := exptypes.ParseAnnotationKey(k)
+			if !ok {
+				continue
+			}
+			if err != nil {
+				return nil, err
+			}
+
+			switch ak.Type {
+			case exptypes.AnnotationIndex, exptypes.AnnotationManifestDescriptor:
+				annotations[ak] = v
+			}
 		}
 	}
-	return filteredAnnotations
+	return annotations, nil
 }
 
 func pushWithMoby(ctx context.Context, d *driver.DriverHandle, name string, l progress.SubLogger) error {
diff --git a/controller/build/build.go b/controller/build/build.go
index 57b4353c..2118d7da 100644
--- a/controller/build/build.go
+++ b/controller/build/build.go
@@ -147,8 +147,6 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in controllerapi.Build
 
 	opts.Exports = outputs
 
-	opts.Annotations = annotations
-
 	opts.CacheFrom = controllerapi.CreateCaches(in.CacheFrom)
 	opts.CacheTo = controllerapi.CreateCaches(in.CacheTo)