bake: new hclparser package

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi
2021-06-23 18:03:17 -08:00
parent 03b7128b60
commit c5eb8f58b4
8 changed files with 1010 additions and 711 deletions

153
bake/hclparser/expr.go Normal file
View File

@@ -0,0 +1,153 @@
package hclparser
import (
"reflect"
"unsafe"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/pkg/errors"
)
func funcCalls(exp hcl.Expression) ([]string, hcl.Diagnostics) {
node, ok := exp.(hclsyntax.Node)
if !ok {
fns, err := jsonFuncCallsRecursive(exp)
if err != nil {
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: err.Error(),
Subject: exp.Range().Ptr(),
Context: exp.Range().Ptr(),
},
}
}
return fns, nil
}
var funcnames []string
hcldiags := hclsyntax.VisitAll(node, func(n hclsyntax.Node) hcl.Diagnostics {
if fe, ok := n.(*hclsyntax.FunctionCallExpr); ok {
funcnames = append(funcnames, fe.Name)
}
return nil
})
if hcldiags.HasErrors() {
return nil, hcldiags
}
return funcnames, nil
}
func jsonFuncCallsRecursive(exp hcl.Expression) ([]string, error) {
je, ok := exp.(jsonExp)
if !ok {
return nil, errors.Errorf("invalid expression type %T", exp)
}
m := map[string]struct{}{}
for _, e := range elementExpressions(je, exp) {
if err := appendJSONFuncCalls(e, m); err != nil {
return nil, err
}
}
arr := make([]string, 0, len(m))
for n := range m {
arr = append(arr, n)
}
return arr, nil
}
func appendJSONFuncCalls(exp hcl.Expression, m map[string]struct{}) error {
v := reflect.ValueOf(exp)
if v.Kind() != reflect.Ptr || v.IsNil() {
return errors.Errorf("invalid json expression kind %T %v", exp, v.Kind())
}
src := v.Elem().FieldByName("src")
if src.IsZero() {
return errors.Errorf("%v has no property src", v.Elem().Type())
}
if src.Kind() != reflect.Interface {
return errors.Errorf("%v src is not interface: %v", src.Type(), src.Kind())
}
src = src.Elem()
if src.IsNil() {
return nil
}
if src.Kind() == reflect.Ptr {
src = src.Elem()
}
if src.Kind() != reflect.Struct {
return errors.Errorf("%v is not struct: %v", src.Type(), src.Kind())
}
// hcl/v2/json/ast#stringVal
val := src.FieldByName("Value")
if val.IsZero() {
return nil
}
rng := src.FieldByName("SrcRange")
if val.IsZero() {
return nil
}
var stringVal struct {
Value string
SrcRange hcl.Range
}
if !val.Type().AssignableTo(reflect.ValueOf(stringVal.Value).Type()) {
return nil
}
if !rng.Type().AssignableTo(reflect.ValueOf(stringVal.SrcRange).Type()) {
return nil
}
// reflect.Set does not work for unexported fields
stringVal.Value = *(*string)(unsafe.Pointer(val.UnsafeAddr()))
stringVal.SrcRange = *(*hcl.Range)(unsafe.Pointer(rng.UnsafeAddr()))
expr, diags := hclsyntax.ParseExpression([]byte(stringVal.Value), stringVal.SrcRange.Filename, stringVal.SrcRange.Start)
if diags.HasErrors() {
return nil
}
fns, err := funcCalls(expr)
if err != nil {
return err
}
for _, fn := range fns {
m[fn] = struct{}{}
}
return nil
}
type jsonExp interface {
ExprList() []hcl.Expression
ExprMap() []hcl.KeyValuePair
}
func elementExpressions(je jsonExp, exp hcl.Expression) []hcl.Expression {
list := je.ExprList()
if len(list) != 0 {
exp := make([]hcl.Expression, 0, len(list))
for _, e := range list {
if je, ok := e.(jsonExp); ok {
exp = append(exp, elementExpressions(je, e)...)
}
}
return exp
}
kvlist := je.ExprMap()
if len(kvlist) != 0 {
exp := make([]hcl.Expression, 0, len(kvlist)*2)
for _, p := range kvlist {
exp = append(exp, p.Key)
if je, ok := p.Value.(jsonExp); ok {
exp = append(exp, elementExpressions(je, p.Value)...)
}
}
return exp
}
return []hcl.Expression{exp}
}

474
bake/hclparser/hclparser.go Normal file
View File

@@ -0,0 +1,474 @@
package hclparser
import (
"fmt"
"math"
"math/big"
"reflect"
"strconv"
"strings"
"github.com/docker/buildx/util/userfunc"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/pkg/errors"
"github.com/zclconf/go-cty/cty"
)
type Opt struct {
LookupVar func(string) (string, bool)
}
type variable struct {
Name string `json:"-" hcl:"name,label"`
Default *hcl.Attribute `json:"default,omitempty" hcl:"default,optional"`
Body hcl.Body `json:"-" hcl:",body"`
}
type functionDef struct {
Name string `json:"-" hcl:"name,label"`
Params *hcl.Attribute `json:"params,omitempty" hcl:"params"`
Variadic *hcl.Attribute `json:"variadic_param,omitempty" hcl:"variadic_params"`
Result *hcl.Attribute `json:"result,omitempty" hcl:"result"`
}
type inputs struct {
Variables []*variable `hcl:"variable,block"`
Functions []*functionDef `hcl:"function,block"`
Remain hcl.Body `json:"-" hcl:",remain"`
}
type parser struct {
opt Opt
vars map[string]*variable
attrs map[string]*hcl.Attribute
funcs map[string]*functionDef
ectx *hcl.EvalContext
progress map[string]struct{}
progressF map[string]struct{}
doneF map[string]struct{}
}
func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.Diagnostics {
fns, hcldiags := funcCalls(exp)
if hcldiags.HasErrors() {
return hcldiags
}
for _, fn := range fns {
if err := p.resolveFunction(fn); err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: err.Error(),
Subject: exp.Range().Ptr(),
Context: exp.Range().Ptr(),
},
}
}
}
for _, v := range exp.Variables() {
if _, ok := exclude[v.RootName()]; ok {
continue
}
if err := p.resolveValue(v.RootName()); err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: err.Error(),
Subject: v.SourceRange().Ptr(),
Context: v.SourceRange().Ptr(),
},
}
}
}
return nil
}
func (p *parser) resolveFunction(name string) error {
if _, ok := p.doneF[name]; ok {
return nil
}
f, ok := p.funcs[name]
if !ok {
if _, ok := p.ectx.Functions[name]; ok {
return nil
}
return errors.Errorf("undefined function %s", name)
}
if _, ok := p.progressF[name]; ok {
return errors.Errorf("function cycle not allowed for %s", name)
}
p.progressF[name] = struct{}{}
paramExprs, paramsDiags := hcl.ExprList(f.Params.Expr)
if paramsDiags.HasErrors() {
return paramsDiags
}
var diags hcl.Diagnostics
params := map[string]struct{}{}
for _, paramExpr := range paramExprs {
param := hcl.ExprAsKeyword(paramExpr)
if param == "" {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid param element",
Detail: "Each parameter name must be an identifier.",
Subject: paramExpr.Range().Ptr(),
})
}
params[param] = struct{}{}
}
var variadic hcl.Expression
if f.Variadic != nil {
variadic = f.Variadic.Expr
param := hcl.ExprAsKeyword(variadic)
if param == "" {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid param element",
Detail: "Each parameter name must be an identifier.",
Subject: f.Variadic.Range.Ptr(),
})
}
params[param] = struct{}{}
}
if diags.HasErrors() {
return diags
}
if diags := p.loadDeps(f.Result.Expr, params); diags.HasErrors() {
return diags
}
v, diags := userfunc.NewFunction(f.Params.Expr, variadic, f.Result.Expr, func() *hcl.EvalContext {
return p.ectx
})
if diags.HasErrors() {
return diags
}
p.doneF[name] = struct{}{}
p.ectx.Functions[name] = v
return nil
}
func (p *parser) resolveValue(name string) (err error) {
if _, ok := p.ectx.Variables[name]; ok {
return nil
}
if _, ok := p.progress[name]; ok {
return errors.Errorf("variable cycle not allowed for %s", name)
}
p.progress[name] = struct{}{}
var v *cty.Value
defer func() {
if v != nil {
p.ectx.Variables[name] = *v
}
}()
def, ok := p.attrs[name]
if !ok {
vr, ok := p.vars[name]
if !ok {
return errors.Errorf("undefined variable %q", name)
}
def = vr.Default
}
if def == nil {
val, _ := p.opt.LookupVar(name)
vv := cty.StringVal(val)
v = &vv
return
}
if diags := p.loadDeps(def.Expr, nil); diags.HasErrors() {
return diags
}
vv, diags := def.Expr.Value(p.ectx)
if diags.HasErrors() {
return diags
}
_, isVar := p.vars[name]
if envv, ok := p.opt.LookupVar(name); ok && isVar {
if vv.Type().Equals(cty.Bool) {
b, err := strconv.ParseBool(envv)
if err != nil {
return errors.Wrapf(err, "failed to parse %s as bool", name)
}
vv := cty.BoolVal(b)
v = &vv
return nil
} else if vv.Type().Equals(cty.String) {
vv := cty.StringVal(envv)
v = &vv
return nil
} else if vv.Type().Equals(cty.Number) {
n, err := strconv.ParseFloat(envv, 64)
if err == nil && (math.IsNaN(n) || math.IsInf(n, 0)) {
err = errors.Errorf("invalid number value")
}
if err != nil {
return errors.Wrapf(err, "failed to parse %s as number", name)
}
vv := cty.NumberVal(big.NewFloat(n))
v = &vv
return nil
} else {
// TODO: support lists with csv values
return errors.Errorf("unsupported type %s for variable %s", v.Type(), name)
}
}
v = &vv
return nil
}
func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
reserved := map[string]struct{}{}
schema, _ := gohcl.ImpliedBodySchema(val)
for _, bs := range schema.Blocks {
reserved[bs.Type] = struct{}{}
}
var defs inputs
if err := gohcl.DecodeBody(b, nil, &defs); err != nil {
return err
}
if opt.LookupVar == nil {
opt.LookupVar = func(string) (string, bool) {
return "", false
}
}
p := &parser{
opt: opt,
vars: map[string]*variable{},
attrs: map[string]*hcl.Attribute{},
funcs: map[string]*functionDef{},
progress: map[string]struct{}{},
progressF: map[string]struct{}{},
doneF: map[string]struct{}{},
ectx: &hcl.EvalContext{
Variables: map[string]cty.Value{},
Functions: stdlibFunctions,
},
}
for _, v := range defs.Variables {
// TODO: validate name
if _, ok := reserved[v.Name]; ok {
continue
}
p.vars[v.Name] = v
}
for _, v := range defs.Functions {
// TODO: validate name
if _, ok := reserved[v.Name]; ok {
continue
}
p.funcs[v.Name] = v
}
attrs, diags := b.JustAttributes()
if diags.HasErrors() {
for _, d := range diags {
if d.Detail != "Blocks are not allowed here." {
return diags
}
}
}
for _, v := range attrs {
if _, ok := reserved[v.Name]; ok {
continue
}
p.attrs[v.Name] = v
}
delete(p.attrs, "function")
for k := range p.attrs {
if err := p.resolveValue(k); err != nil {
if diags, ok := err.(hcl.Diagnostics); ok {
return diags
}
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid attribute",
Detail: err.Error(),
Subject: &p.attrs[k].Range,
Context: &p.attrs[k].Range,
},
}
}
}
for k := range p.vars {
if err := p.resolveValue(k); err != nil {
if diags, ok := err.(hcl.Diagnostics); ok {
return diags
}
r := p.vars[k].Body.MissingItemRange()
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid value",
Detail: err.Error(),
Subject: &r,
Context: &r,
},
}
}
}
for k := range p.funcs {
if err := p.resolveFunction(k); err != nil {
if diags, ok := err.(hcl.Diagnostics); ok {
return diags
}
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid function",
Detail: err.Error(),
Subject: &p.funcs[k].Params.Range,
Context: &p.funcs[k].Params.Range,
},
}
}
}
content, _, diags := b.PartialContent(schema)
if diags.HasErrors() {
return diags
}
for _, a := range content.Attributes {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid attribute",
Detail: "global attributes currently not supported",
Subject: &a.Range,
Context: &a.Range,
},
}
}
m := map[string]map[string][]*hcl.Block{}
for _, b := range content.Blocks {
if len(b.Labels) == 0 || len(b.Labels) > 1 {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid block",
Detail: fmt.Sprintf("invalid block label: %v", b.Labels),
Subject: &b.LabelRanges[0],
Context: &b.LabelRanges[0],
},
}
}
bm, ok := m[b.Type]
if !ok {
bm = map[string][]*hcl.Block{}
m[b.Type] = bm
}
lbl := b.Labels[0]
bm[lbl] = append(bm[lbl], b)
}
vt := reflect.ValueOf(val).Elem().Type()
numFields := vt.NumField()
type value struct {
reflect.Value
idx int
}
type field struct {
idx int
typ reflect.Type
values map[string]value
}
types := map[string]field{}
for i := 0; i < numFields; i++ {
tags := strings.Split(vt.Field(i).Tag.Get("hcl"), ",")
types[tags[0]] = field{
idx: i,
typ: vt.Field(i).Type,
values: make(map[string]value),
}
}
diags = hcl.Diagnostics{}
for _, b := range content.Blocks {
v := reflect.ValueOf(val)
t, ok := types[b.Type]
if !ok {
continue
}
vv := reflect.New(t.typ.Elem().Elem())
diag := gohcl.DecodeBody(b.Body, p.ectx, vv.Interface())
if diag.HasErrors() {
diags = append(diags, diag...)
continue
}
setLabel(vv, b.Labels[0])
oldValue, exists := t.values[b.Labels[0]]
if exists {
if m := oldValue.Value.MethodByName("Merge"); m.IsValid() {
m.Call([]reflect.Value{vv})
} else {
v.Elem().Field(t.idx).Index(oldValue.idx).Set(vv)
}
} else {
slice := v.Elem().Field(t.idx)
if slice.IsNil() {
slice = reflect.New(t.typ).Elem()
}
t.values[b.Labels[0]] = value{Value: vv, idx: slice.Len()}
v.Elem().Field(t.idx).Set(reflect.Append(slice, vv))
}
}
if diags.HasErrors() {
return diags
}
return nil
}
func setLabel(v reflect.Value, lbl string) {
// cache field index?
numFields := v.Elem().Type().NumField()
for i := 0; i < numFields; i++ {
for _, t := range strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") {
if t == "label" {
v.Elem().Field(i).Set(reflect.ValueOf(lbl))
return
}
}
}
}

111
bake/hclparser/stdlib.go Normal file
View File

@@ -0,0 +1,111 @@
package hclparser
import (
"github.com/hashicorp/go-cty-funcs/cidr"
"github.com/hashicorp/go-cty-funcs/crypto"
"github.com/hashicorp/go-cty-funcs/encoding"
"github.com/hashicorp/go-cty-funcs/uuid"
"github.com/hashicorp/hcl/v2/ext/tryfunc"
"github.com/hashicorp/hcl/v2/ext/typeexpr"
"github.com/zclconf/go-cty/cty/function"
"github.com/zclconf/go-cty/cty/function/stdlib"
)
var stdlibFunctions = map[string]function.Function{
"absolute": stdlib.AbsoluteFunc,
"add": stdlib.AddFunc,
"and": stdlib.AndFunc,
"base64decode": encoding.Base64DecodeFunc,
"base64encode": encoding.Base64EncodeFunc,
"bcrypt": crypto.BcryptFunc,
"byteslen": stdlib.BytesLenFunc,
"bytesslice": stdlib.BytesSliceFunc,
"can": tryfunc.CanFunc,
"ceil": stdlib.CeilFunc,
"chomp": stdlib.ChompFunc,
"chunklist": stdlib.ChunklistFunc,
"cidrhost": cidr.HostFunc,
"cidrnetmask": cidr.NetmaskFunc,
"cidrsubnet": cidr.SubnetFunc,
"cidrsubnets": cidr.SubnetsFunc,
"csvdecode": stdlib.CSVDecodeFunc,
"coalesce": stdlib.CoalesceFunc,
"coalescelist": stdlib.CoalesceListFunc,
"compact": stdlib.CompactFunc,
"concat": stdlib.ConcatFunc,
"contains": stdlib.ContainsFunc,
"convert": typeexpr.ConvertFunc,
"distinct": stdlib.DistinctFunc,
"divide": stdlib.DivideFunc,
"element": stdlib.ElementFunc,
"equal": stdlib.EqualFunc,
"flatten": stdlib.FlattenFunc,
"floor": stdlib.FloorFunc,
"formatdate": stdlib.FormatDateFunc,
"format": stdlib.FormatFunc,
"formatlist": stdlib.FormatListFunc,
"greaterthan": stdlib.GreaterThanFunc,
"greaterthanorequalto": stdlib.GreaterThanOrEqualToFunc,
"hasindex": stdlib.HasIndexFunc,
"indent": stdlib.IndentFunc,
"index": stdlib.IndexFunc,
"int": stdlib.IntFunc,
"jsondecode": stdlib.JSONDecodeFunc,
"jsonencode": stdlib.JSONEncodeFunc,
"keys": stdlib.KeysFunc,
"join": stdlib.JoinFunc,
"length": stdlib.LengthFunc,
"lessthan": stdlib.LessThanFunc,
"lessthanorequalto": stdlib.LessThanOrEqualToFunc,
"log": stdlib.LogFunc,
"lookup": stdlib.LookupFunc,
"lower": stdlib.LowerFunc,
"max": stdlib.MaxFunc,
"md5": crypto.Md5Func,
"merge": stdlib.MergeFunc,
"min": stdlib.MinFunc,
"modulo": stdlib.ModuloFunc,
"multiply": stdlib.MultiplyFunc,
"negate": stdlib.NegateFunc,
"notequal": stdlib.NotEqualFunc,
"not": stdlib.NotFunc,
"or": stdlib.OrFunc,
"parseint": stdlib.ParseIntFunc,
"pow": stdlib.PowFunc,
"range": stdlib.RangeFunc,
"regexall": stdlib.RegexAllFunc,
"regex": stdlib.RegexFunc,
"regex_replace": stdlib.RegexReplaceFunc,
"reverse": stdlib.ReverseFunc,
"reverselist": stdlib.ReverseListFunc,
"rsadecrypt": crypto.RsaDecryptFunc,
"sethaselement": stdlib.SetHasElementFunc,
"setintersection": stdlib.SetIntersectionFunc,
"setproduct": stdlib.SetProductFunc,
"setsubtract": stdlib.SetSubtractFunc,
"setsymmetricdifference": stdlib.SetSymmetricDifferenceFunc,
"setunion": stdlib.SetUnionFunc,
"sha1": crypto.Sha1Func,
"sha256": crypto.Sha256Func,
"sha512": crypto.Sha512Func,
"signum": stdlib.SignumFunc,
"slice": stdlib.SliceFunc,
"sort": stdlib.SortFunc,
"split": stdlib.SplitFunc,
"strlen": stdlib.StrlenFunc,
"substr": stdlib.SubstrFunc,
"subtract": stdlib.SubtractFunc,
"timeadd": stdlib.TimeAddFunc,
"title": stdlib.TitleFunc,
"trim": stdlib.TrimFunc,
"trimprefix": stdlib.TrimPrefixFunc,
"trimspace": stdlib.TrimSpaceFunc,
"trimsuffix": stdlib.TrimSuffixFunc,
"try": tryfunc.TryFunc,
"upper": stdlib.UpperFunc,
"urlencode": encoding.URLEncodeFunc,
"uuidv4": uuid.V4Func,
"uuidv5": uuid.V5Func,
"values": stdlib.ValuesFunc,
"zipmap": stdlib.ZipmapFunc,
}