Skip to content

Commit 730b014

Browse files
petroprotsakhpetroprotsakh
authored
SCALRCORE-30060 Pygohcl > Add variable validation (#24)
* SCALRCORE-30060 Add methods for tfvars parsing and variable validation condition evaluation * SCALRCORE-30060 Add docstrings, tests * SCALRCORE-30060 Fix docstrings * SCALRCORE-30060 Fix docstrings * SCALRCORE-30060 Tests: add missing function name assertion
1 parent d145a99 commit 730b014

File tree

5 files changed

+486
-14
lines changed

5 files changed

+486
-14
lines changed

‎pygohcl.go

+239-14
Original file line numberDiff line numberDiff line change
@@ -8,42 +8,267 @@ import "C"
88
import (
99
"encoding/json"
1010
"fmt"
11-
"strings"
12-
11+
"github.com/hashicorp/hcl/v2"
12+
"github.com/hashicorp/hcl/v2/ext/tryfunc"
1313
"github.com/hashicorp/hcl/v2/hclparse"
14+
"github.com/hashicorp/hcl/v2/hclsyntax"
15+
"github.com/zclconf/go-cty/cty"
16+
"github.com/zclconf/go-cty/cty/convert"
17+
"github.com/zclconf/go-cty/cty/function"
18+
"github.com/zclconf/go-cty/cty/function/stdlib"
19+
"strings"
1420
)
1521

1622
//export Parse
1723
func Parse(a *C.char) (resp C.parseResponse) {
18-
defer func() {
19-
if err := recover(); err != nil {
24+
defer func() {
25+
if err := recover(); err != nil {
2026
retValue := fmt.Sprintf("panic HCL: %v", err)
2127
resp = C.parseResponse{nil, C.CString(retValue)}
22-
}
23-
}()
28+
}
29+
}()
2430

2531
input := C.GoString(a)
2632
hclFile, diags := hclparse.NewParser().ParseHCL([]byte(input), "tmp.hcl")
2733
if diags.HasErrors() {
28-
errors := make([]string, 0, len(diags))
29-
for _, diag := range diags {
30-
errors = append(errors, diag.Error())
31-
}
32-
33-
return C.parseResponse{nil, C.CString(fmt.Sprintf("invalid HCL: %s", strings.Join(errors, ", ")))}
34+
return C.parseResponse{nil, C.CString(diagErrorsToString(diags, "invalid HCL: %s"))}
3435
}
3536
hclMap, err := convertFile(hclFile)
3637
if err != nil {
3738
return C.parseResponse{nil, C.CString(fmt.Sprintf("cannot convert HCL to Go map representation: %s", err))}
3839
}
3940
hclInJson, err := json.Marshal(hclMap)
4041
if err != nil {
41-
return C.parseResponse{nil, C.CString(fmt.Sprintf("cannot Go map representation to JSON: %s", err))}
42+
return C.parseResponse{nil, C.CString(fmt.Sprintf("cannot convert Go map representation to JSON: %s", err))}
4243
}
4344
resp = C.parseResponse{C.CString(string(hclInJson)), nil}
4445

4546
return
4647
}
4748

48-
func main() {
49+
//export ParseAttributes
50+
func ParseAttributes(a *C.char) (resp C.parseResponse) {
51+
defer func() {
52+
if err := recover(); err != nil {
53+
retValue := fmt.Sprintf("panic HCL: %v", err)
54+
resp = C.parseResponse{nil, C.CString(retValue)}
55+
}
56+
}()
57+
58+
input := C.GoString(a)
59+
hclFile, parseDiags := hclsyntax.ParseConfig([]byte(input), "tmp.hcl", hcl.InitialPos)
60+
if parseDiags.HasErrors() {
61+
return C.parseResponse{nil, C.CString(diagErrorsToString(parseDiags, "invalid HCL: %s"))}
62+
}
63+
64+
var diags hcl.Diagnostics
65+
hclMap := make(jsonObj)
66+
c := converter{}
67+
68+
attrs, attrsDiags := hclFile.Body.JustAttributes()
69+
diags = diags.Extend(attrsDiags)
70+
71+
for _, attr := range attrs {
72+
_, valueDiags := attr.Expr.Value(nil)
73+
diags = diags.Extend(valueDiags)
74+
if valueDiags.HasErrors() {
75+
continue
76+
}
77+
78+
value, err := c.convertExpression(attr.Expr.(hclsyntax.Expression))
79+
if err != nil {
80+
diags.Append(&hcl.Diagnostic{
81+
Severity: hcl.DiagError,
82+
Summary: "Error processing variable value",
83+
Detail: fmt.Sprintf("Cannot convert HCL to Go map representation: %s.", err),
84+
Subject: attr.NameRange.Ptr(),
85+
})
86+
continue
87+
}
88+
89+
hclMap[attr.Name] = value
90+
}
91+
92+
hclInJson, err := json.Marshal(hclMap)
93+
if err != nil {
94+
diags.Append(&hcl.Diagnostic{
95+
Severity: hcl.DiagError,
96+
Summary: "Error preparing JSON result",
97+
Detail: fmt.Sprintf("Cannot convert Go map representation to JSON: %s.", err),
98+
})
99+
return C.parseResponse{nil, C.CString(diagErrorsToString(diags, ""))}
100+
}
101+
if diags.HasErrors() {
102+
resp = C.parseResponse{C.CString(string(hclInJson)), C.CString(diagErrorsToString(diags, ""))}
103+
} else {
104+
resp = C.parseResponse{C.CString(string(hclInJson)), nil}
105+
}
106+
107+
return
108+
}
109+
110+
//export EvalValidationRule
111+
func EvalValidationRule(c *C.char, e *C.char, n *C.char, v *C.char) (resp *C.char) {
112+
defer func() {
113+
if err := recover(); err != nil {
114+
retValue := fmt.Sprintf("panic HCL: %v", err)
115+
resp = C.CString(retValue)
116+
}
117+
}()
118+
119+
condition := C.GoString(c)
120+
errorMsg := C.GoString(e)
121+
varName := C.GoString(n)
122+
varValue := C.GoString(v)
123+
124+
// First evaluate variable value to get its cty representation
125+
126+
varValueCty, diags := expressionValue(varValue, nil)
127+
if diags.HasErrors() {
128+
if containsError(diags, "Variables not allowed") {
129+
// Try again to handle the case when a string value was provided without enclosing quotes
130+
varValueCty, diags = expressionValue(fmt.Sprintf("%q", varValue), nil)
131+
}
132+
}
133+
if diags.HasErrors() {
134+
return C.CString(diagErrorsToString(diags, "cannot process variable value: %s"))
135+
}
136+
137+
// Now evaluate the condition
138+
139+
hclCtx := &hcl.EvalContext{
140+
Variables: map[string]cty.Value{
141+
"var": cty.ObjectVal(map[string]cty.Value{
142+
varName: varValueCty,
143+
}),
144+
},
145+
Functions: knownFunctions,
146+
}
147+
conditionCty, diags := expressionValue(condition, hclCtx)
148+
if diags.HasErrors() {
149+
return C.CString(diagErrorsToString(diags, "cannot process condition expression: %s"))
150+
}
151+
152+
if conditionCty.IsNull() {
153+
return C.CString("condition expression result is null")
154+
}
155+
156+
conditionCty, err := convert.Convert(conditionCty, cty.Bool)
157+
if err != nil {
158+
return C.CString("condition expression result must be bool")
159+
}
160+
161+
if conditionCty.True() {
162+
return nil
163+
}
164+
165+
// Finally evaluate the error message expression
166+
167+
var errorMsgValue = "cannot process error message expression"
168+
errorMsgCty, diags := expressionValue(errorMsg, hclCtx)
169+
if diags.HasErrors() {
170+
errorMsgCty, diags = expressionValue(fmt.Sprintf("%q", errorMsg), hclCtx)
171+
}
172+
if !diags.HasErrors() && !errorMsgCty.IsNull() {
173+
errorMsgCty, err = convert.Convert(errorMsgCty, cty.String)
174+
if err == nil {
175+
errorMsgValue = errorMsgCty.AsString()
176+
}
177+
}
178+
return C.CString(errorMsgValue)
179+
}
180+
181+
func diagErrorsToString(diags hcl.Diagnostics, format string) string {
182+
diagErrs := diags.Errs()
183+
errors := make([]string, 0, len(diagErrs))
184+
for _, err := range diagErrs {
185+
errors = append(errors, err.Error())
186+
}
187+
if format == "" {
188+
return strings.Join(errors, ", ")
189+
}
190+
return fmt.Sprintf(format, strings.Join(errors, ", "))
191+
}
192+
193+
func containsError(diags hcl.Diagnostics, e string) bool {
194+
for _, err := range diags.Errs() {
195+
if strings.Contains(err.Error(), e) {
196+
return true
197+
}
198+
}
199+
return false
49200
}
201+
202+
func expressionValue(in string, ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
203+
var diags hcl.Diagnostics
204+
205+
expr, diags := hclsyntax.ParseExpression([]byte(in), "tmp.hcl", hcl.InitialPos)
206+
if diags.HasErrors() {
207+
return cty.NilVal, diags
208+
}
209+
210+
val, diags := expr.Value(ctx)
211+
if diags.HasErrors() {
212+
return cty.NilVal, diags
213+
}
214+
215+
return val, diags
216+
}
217+
218+
var knownFunctions = map[string]function.Function{
219+
"abs": stdlib.AbsoluteFunc,
220+
"can": tryfunc.CanFunc,
221+
"ceil": stdlib.CeilFunc,
222+
"chomp": stdlib.ChompFunc,
223+
"coalescelist": stdlib.CoalesceListFunc,
224+
"compact": stdlib.CompactFunc,
225+
"concat": stdlib.ConcatFunc,
226+
"contains": stdlib.ContainsFunc,
227+
"csvdecode": stdlib.CSVDecodeFunc,
228+
"distinct": stdlib.DistinctFunc,
229+
"element": stdlib.ElementFunc,
230+
"chunklist": stdlib.ChunklistFunc,
231+
"flatten": stdlib.FlattenFunc,
232+
"floor": stdlib.FloorFunc,
233+
"format": stdlib.FormatFunc,
234+
"formatdate": stdlib.FormatDateFunc,
235+
"formatlist": stdlib.FormatListFunc,
236+
"indent": stdlib.IndentFunc,
237+
"join": stdlib.JoinFunc,
238+
"jsondecode": stdlib.JSONDecodeFunc,
239+
"jsonencode": stdlib.JSONEncodeFunc,
240+
"keys": stdlib.KeysFunc,
241+
"log": stdlib.LogFunc,
242+
"lower": stdlib.LowerFunc,
243+
"max": stdlib.MaxFunc,
244+
"merge": stdlib.MergeFunc,
245+
"min": stdlib.MinFunc,
246+
"parseint": stdlib.ParseIntFunc,
247+
"pow": stdlib.PowFunc,
248+
"range": stdlib.RangeFunc,
249+
"regex": stdlib.RegexFunc,
250+
"regexall": stdlib.RegexAllFunc,
251+
"reverse": stdlib.ReverseListFunc,
252+
"setintersection": stdlib.SetIntersectionFunc,
253+
"setproduct": stdlib.SetProductFunc,
254+
"setsubtract": stdlib.SetSubtractFunc,
255+
"setunion": stdlib.SetUnionFunc,
256+
"signum": stdlib.SignumFunc,
257+
"slice": stdlib.SliceFunc,
258+
"sort": stdlib.SortFunc,
259+
"split": stdlib.SplitFunc,
260+
"strrev": stdlib.ReverseFunc,
261+
"substr": stdlib.SubstrFunc,
262+
"timeadd": stdlib.TimeAddFunc,
263+
"title": stdlib.TitleFunc,
264+
"trim": stdlib.TrimFunc,
265+
"trimprefix": stdlib.TrimPrefixFunc,
266+
"trimspace": stdlib.TrimSpaceFunc,
267+
"trimsuffix": stdlib.TrimSuffixFunc,
268+
"try": tryfunc.TryFunc,
269+
"upper": stdlib.UpperFunc,
270+
"values": stdlib.ValuesFunc,
271+
"zipmap": stdlib.ZipmapFunc,
272+
}
273+
274+
func main() {}

‎pygohcl/__init__.py

+90
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ class HCLInternalError(Exception):
2424
pass
2525

2626

27+
class ValidationError(Exception):
28+
pass
29+
30+
31+
class UnknownFunctionError(ValidationError):
32+
pass
33+
34+
2735
def loadb(data: bytes) -> tp.Dict:
2836
s = ffi.new("char[]", data)
2937
ret = lib.Parse(s)
@@ -46,3 +54,85 @@ def loads(data: str) -> tp.Dict:
4654
def load(stream: tp.IO) -> tp.Dict:
4755
data = stream.read()
4856
return loadb(data)
57+
58+
59+
def attributes_loadb(data: bytes) -> tp.Dict:
60+
"""
61+
Like :func:`pygohcl.loadb`,
62+
but expects from the input to contain only top-level attributes.
63+
64+
Example:
65+
>>> hcl = '''
66+
... key1 = "value"
67+
... key2 = false
68+
... key3 = [1, 2, 3]
69+
... '''
70+
>>> import pygohcl
71+
>>> print(pygohcl.attributes_loads(hcl))
72+
{'key1': 'value', 'key2': False, 'key3': [1, 2, 3]}
73+
74+
:raises HCLParseError: when the provided input cannot be parsed as valid HCL,
75+
or it contains other blocks, not only attributes.
76+
"""
77+
s = ffi.new("char[]", data)
78+
ret = lib.ParseAttributes(s)
79+
if ret.err != ffi.NULL:
80+
err: bytes = ffi.string(ret.err)
81+
ffi.gc(ret.err, lib.free)
82+
err = err.decode("utf8")
83+
raise HCLParseError(err)
84+
ret_json = ffi.string(ret.json)
85+
ffi.gc(ret.json, lib.free)
86+
return json.loads(ret_json)
87+
88+
89+
def attributes_loads(data: str) -> tp.Dict:
90+
return attributes_loadb(data.encode("utf8"))
91+
92+
93+
def attributes_load(stream: tp.IO) -> tp.Dict:
94+
data = stream.read()
95+
return attributes_loadb(data)
96+
97+
98+
def eval_var_condition(
99+
condition: str, error_message: str, variable_name: str, variable_value: str
100+
) -> None:
101+
"""
102+
This is specific to Terraform/OpenTofu configuration language
103+
and is meant to evaluate results of the `validation` block of a variable definition.
104+
105+
This comes with a limited selection of supported functions.
106+
Terraform/OpenTofu expand this list with their own set
107+
of useful functions, which will not pass this validation.
108+
For that reason a separate `UnknownFunctionError` is raised then,
109+
so the consumer can decide how to treat this case.
110+
111+
Example:
112+
>>> import pygohcl
113+
>>> pygohcl.eval_var_condition(
114+
... condition="var.count < 3",
115+
... error_message="count must be less than 3, but ${var.count} was given",
116+
... variable_name="count",
117+
... variable_value="5",
118+
... )
119+
Traceback (most recent call last):
120+
...
121+
pygohcl.ValidationError: count must be less than 3, but 5 was given
122+
123+
:raises ValidationError: when the condition expression has not evaluated to `True`
124+
:raises UnknownFunctionError: when the condition expression refers to a function
125+
that is not known to the library
126+
"""
127+
c = ffi.new("char[]", condition.encode("utf8"))
128+
e = ffi.new("char[]", error_message.encode("utf8"))
129+
n = ffi.new("char[]", variable_name.encode("utf8"))
130+
v = ffi.new("char[]", variable_value.encode("utf8"))
131+
ret = lib.EvalValidationRule(c, e, n, v)
132+
if ret != ffi.NULL:
133+
err: bytes = ffi.string(ret)
134+
ffi.gc(ret, lib.free)
135+
err = err.decode("utf8")
136+
if "Call to unknown function" in err:
137+
raise UnknownFunctionError(err)
138+
raise ValidationError(err)

‎pygohcl/build_cffi.py

+2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
} parseResponse;
1919
2020
parseResponse Parse(char* a);
21+
parseResponse ParseAttributes(char* a);
22+
char* EvalValidationRule(char* c, char* e, char* n, char* v);
2123
void free(void *ptr);
2224
"""
2325
)

0 commit comments

Comments
 (0)