docker/buildx
1
0
Fork 0
mirror of https://gitea.com/Lydanne/buildx.git synced 2025-07-12 22:47:09 +08:00
Code Issues Packages Projects Releases 1 Wiki Activity

feat(driver/kubernetes): support mount buildkit.toml and qemu installing

Browse Source 
Signed-off-by: Morlay <morlay.null@gmail.com>
This commit is contained in:
Morlay
2021-07-21 10:39:58 +08:00
parent cd133cee25
commit c76b5eac03
8 changed files with 208 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
driver/kubernetes/driver.go
View File

@ -18,6 +18,8 @@ import (
"github.com/moby/buildkit/util/tracing/detect"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
clientappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
@ -39,15 +41,18 @@ type Driver struct {
factory driver.Factory
minReplicas int
deployment *appsv1.Deployment
configMap *corev1.ConfigMap
clientset *kubernetes.Clientset
deploymentClient clientappsv1.DeploymentInterface
podClient clientcorev1.PodInterface
configMapClient clientcorev1.ConfigMapInterface
podChooser podchooser.PodChooser
}
func (d *Driver) IsMobyDriver() bool {
return false
}
func (d *Driver) Config() driver.InitConfig {
return d.InitConfig
}
@ -56,7 +61,24 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error {
_, err := d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})
if err != nil {
// TODO: return err if err != ErrNotFound
if !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "error for bootstrap %q", d.deployment.Name)
}
if d.configMap != nil {
// create ConfigMap first if exists
_, err = d.configMapClient.Create(ctx, d.configMap, metav1.CreateOptions{})
if err != nil {
if !apierrors.IsAlreadyExists(err) {
return errors.Wrapf(err, "error while calling configMapClient.Create for %q", d.configMap.Name)
}
_, err = d.configMapClient.Update(ctx, d.configMap, metav1.UpdateOptions{})
if err != nil {
return errors.Wrapf(err, "error while calling configMapClient.Update for %q", d.configMap.Name)
}
}
}
_, err = d.deploymentClient.Create(ctx, d.deployment, metav1.CreateOptions{})
if err != nil {
return errors.Wrapf(err, "error while calling deploymentClient.Create for %q", d.deployment.Name)
@ -145,7 +167,16 @@ func (d *Driver) Stop(ctx context.Context, force bool) error {
func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
if err := d.deploymentClient.Delete(ctx, d.deployment.Name, metav1.DeleteOptions{}); err != nil {
return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
if !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
}
}
if d.configMap != nil {
if err := d.configMapClient.Delete(ctx, d.configMap.Name, metav1.DeleteOptions{}); err != nil {
if !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", d.configMap.Name)
}
}
}
return nil
}

39
driver/kubernetes/factory.go
View File

@ -2,6 +2,7 @@ package kubernetes
import (
"context"
"os"
"strconv"
"strings"
@ -59,11 +60,13 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
if err != nil {
return nil, err
}
d := &Driver{
factory: f,
InitConfig: cfg,
clientset: clientset,
}
deploymentOpt := &manifest.DeploymentOpt{
Name: deploymentName,
Image: bkimage.DefaultImage,
@ -72,12 +75,25 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
Rootless: false,
Platforms: cfg.Platforms,
}
deploymentOpt.Qemu.Image = bkimage.QemuImage
if cfg.ConfigFile != "" {
buildkitConfig, err := os.ReadFile(cfg.ConfigFile)
if err != nil {
return nil, err
}
deploymentOpt.BuildkitConfig = buildkitConfig
}
loadbalance := LoadbalanceSticky
imageOverride := ""
for k, v := range cfg.DriverOpts {
switch k {
case "image":
imageOverride = v
if v != "" {
deploymentOpt.Image = v
}
case "namespace":
namespace = v
case "replicas":
@ -117,20 +133,31 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
return nil, errors.Errorf("invalid loadbalance %q", v)
}
loadbalance = v
case "qemu.install":
deploymentOpt.Qemu.Install, err = strconv.ParseBool(v)
if err != nil {
return nil, err
}
case "qemu.image":
if v != "" {
deploymentOpt.Qemu.Image = v
}
default:
return nil, errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
}
}
if imageOverride != "" {
deploymentOpt.Image = imageOverride
}
d.deployment, err = manifest.NewDeployment(deploymentOpt)
d.deployment, d.configMap, err = manifest.NewDeployment(deploymentOpt)
if err != nil {
return nil, err
}
d.minReplicas = deploymentOpt.Replicas
d.deploymentClient = clientset.AppsV1().Deployments(namespace)
d.podClient = clientset.CoreV1().Pods(namespace)
d.configMapClient = clientset.CoreV1().ConfigMaps(namespace)
switch loadbalance {
case LoadbalanceSticky:
d.podChooser = &podchooser.StickyPodChooser{

85
driver/kubernetes/manifest/manifest.go
View File

@ -12,11 +12,23 @@ import (
)
type DeploymentOpt struct {
Namespace string
Name string
Image string
Replicas int
BuildkitFlags []string
Namespace string
Name string
Image string
Replicas int
// Qemu
Qemu struct {
// when true, will install binfmt
Install bool
Image string
}
BuildkitFlags []string
// BuildkitConfig
// when not empty, will create configmap with buildkit.toml and mounted
BuildkitConfig []byte
Rootless bool
NodeSelector map[string]string
RequestsCPU string
@ -31,7 +43,7 @@ const (
AnnotationPlatform = "buildx.docker.com/platform"
)
func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMap, err error) {
labels := map[string]string{
"app": opt.Name,
}
@ -44,7 +56,7 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
annotations[AnnotationPlatform] = strings.Join(platformutil.Format(opt.Platforms), ",")
}
d := &appsv1.Deployment{
d = &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: "Deployment",
@ -91,9 +103,56 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
},
},
}
if len(opt.BuildkitConfig) > 0 {
c = &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: corev1.SchemeGroupVersion.String(),
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: opt.Namespace,
Name: opt.Name + "-config",
Annotations: annotations,
},
Data: map[string]string{
"buildkitd.toml": string(opt.BuildkitConfig),
},
}
d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{
Name: "config",
MountPath: "/etc/buildkit",
}}
d.Spec.Template.Spec.Volumes = []corev1.Volume{{
Name: "config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: c.Name,
},
},
},
}}
}
if opt.Qemu.Install {
d.Spec.Template.Spec.InitContainers = []corev1.Container{
{
Name: "qemu",
Image: opt.Qemu.Image,
Args: []string{"--install", "all"},
SecurityContext: &corev1.SecurityContext{
Privileged: &privileged,
},
},
}
}
if opt.Rootless {
if err := toRootless(d); err != nil {
return nil, err
return nil, nil, err
}
}
@ -104,7 +163,7 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
if opt.RequestsCPU != "" {
reqCPU, err := resource.ParseQuantity(opt.RequestsCPU)
if err != nil {
return nil, err
return nil, nil, err
}
d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = reqCPU
}
@ -112,7 +171,7 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
if opt.RequestsMemory != "" {
reqMemory, err := resource.ParseQuantity(opt.RequestsMemory)
if err != nil {
return nil, err
return nil, nil, err
}
d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = reqMemory
}
@ -120,7 +179,7 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
if opt.LimitsCPU != "" {
limCPU, err := resource.ParseQuantity(opt.LimitsCPU)
if err != nil {
return nil, err
return nil, nil, err
}
d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU] = limCPU
}
@ -128,12 +187,12 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
if opt.LimitsMemory != "" {
limMemory, err := resource.ParseQuantity(opt.LimitsMemory)
if err != nil {
return nil, err
return nil, nil, err
}
d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = limMemory
}
return d, nil
return
}
func toRootless(d *appsv1.Deployment) error {