Skip to content

Commit d2cb3ad

Browse files
committed
starlarkjson: a standard JSON module for Starlark
This change defines a standard Starlark module for JSON encoding and decoding. See json.go for documentation. It is intended to subsume, generalize, and eventually replace Bazel's ill-conceived struct.to_json method. The json module is predeclared in the Starlark REPL environment. See related issues: bazelbuild/bazel#7896 https://buganizer.corp.google.com/issues/23962735 https://buganizer.corp.google.com/issues/70210417 bazelbuild/bazel#7879 (comment) bazelbuild/bazel#5542 bazelbuild/bazel#10176 bazelbuild/starlark#83 bazelbuild/bazel#3732 Change-Id: I297ffaee9349eedeeb52f5a88f40636a4095f997
1 parent dcffbd0 commit d2cb3ad

File tree

4 files changed

+639
-1
lines changed

4 files changed

+639
-1
lines changed

cmd/starlark/starlark.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"go.starlark.net/repl"
2020
"go.starlark.net/resolve"
2121
"go.starlark.net/starlark"
22+
"go.starlark.net/starlarkjson"
2223
)
2324

2425
// flags
@@ -88,6 +89,10 @@ func doMain() int {
8889
thread := &starlark.Thread{Load: repl.MakeLoad()}
8990
globals := make(starlark.StringDict)
9091

92+
// Ideally this statement would update the predeclared environment.
93+
// TODO(adonovan): plumb predeclared env through to the REPL.
94+
starlark.Universe["json"] = starlarkjson.Module
95+
9196
switch {
9297
case flag.NArg() == 1 || *execprog != "":
9398
var (
@@ -113,7 +118,6 @@ func doMain() int {
113118
fmt.Println("Welcome to Starlark (go.starlark.net)")
114119
thread.Name = "REPL"
115120
repl.REPL(thread, globals)
116-
return 0
117121
default:
118122
log.Print("want at most one Starlark file name")
119123
return 1

starlark/eval_test.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ import (
1616
"go.starlark.net/internal/chunkedfile"
1717
"go.starlark.net/resolve"
1818
"go.starlark.net/starlark"
19+
"go.starlark.net/starlarkjson"
20+
"go.starlark.net/starlarkstruct"
1921
"go.starlark.net/starlarktest"
2022
"go.starlark.net/syntax"
2123
)
@@ -119,6 +121,7 @@ func TestExecFile(t *testing.T) {
119121
"testdata/float.star",
120122
"testdata/function.star",
121123
"testdata/int.star",
124+
"testdata/json.star",
122125
"testdata/list.star",
123126
"testdata/misc.star",
124127
"testdata/set.star",
@@ -132,6 +135,7 @@ func TestExecFile(t *testing.T) {
132135
predeclared := starlark.StringDict{
133136
"hasfields": starlark.NewBuiltin("hasfields", newHasFields),
134137
"fibonacci": fib{},
138+
"struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
135139
}
136140

137141
setOptions(chunk.Source)
@@ -186,6 +190,9 @@ func load(thread *starlark.Thread, module string) (starlark.StringDict, error) {
186190
if module == "assert.star" {
187191
return starlarktest.LoadAssertModule()
188192
}
193+
if module == "json.star" {
194+
return starlark.StringDict{"json": starlarkjson.Module}, nil
195+
}
189196

190197
// TODO(adonovan): test load() using this execution path.
191198
filename := filepath.Join(filepath.Dir(thread.CallFrame(0).Pos.Filename()), module)

starlark/testdata/json.star

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
# Tests of json module.
2+
# option:float
3+
4+
load("assert.star", "assert")
5+
load("json.star", "json")
6+
7+
assert.eq(dir(json), ["decode", "encode", "indent"])
8+
9+
# Some of these cases were inspired by github.com/nst/JSONTestSuite.
10+
11+
## json.encode
12+
13+
assert.eq(json.encode(None), "null")
14+
assert.eq(json.encode(True), "true")
15+
assert.eq(json.encode(False), "false")
16+
assert.eq(json.encode(-123), "-123")
17+
assert.eq(json.encode(12345*12345*12345*12345*12345*12345), "3539537889086624823140625")
18+
assert.eq(json.encode(float(12345*12345*12345*12345*12345*12345)), "3.539537889086625e+24")
19+
assert.eq(json.encode(12.345e67), "1.2345e+68")
20+
assert.eq(json.encode("hello"), '"hello"')
21+
assert.eq(json.encode([1, 2, 3]), "[1,2,3]")
22+
assert.eq(json.encode((1, 2, 3)), "[1,2,3]")
23+
assert.eq(json.encode(range(3)), "[0,1,2]") # a built-in iterable
24+
assert.eq(json.encode(dict(x = 1, y = "two")), '{"x":1,"y":"two"}')
25+
assert.eq(json.encode(struct(x = 1, y = "two")), '{"x":1,"y":"two"}') # a user-defined HasAttrs
26+
assert.eq(json.encode("\x80"), '"\\ufffd"') # invalid UTF-8 -> replacement char
27+
28+
def encode_error(expr, error):
29+
assert.fails(lambda: json.encode(expr), error)
30+
31+
encode_error(float("NaN"), "json.encode: cannot encode non-finite float NaN")
32+
encode_error({1: "two"}, "dict has int key, want string")
33+
encode_error(len, "cannot encode builtin_function_or_method as JSON")
34+
encode_error(struct(x=[1, {"x": len}]), # nested failure
35+
'in field .x: at list index 1: in dict key "x": cannot encode...')
36+
encode_error(struct(x=[1, {"x": len}]), # nested failure
37+
'in field .x: at list index 1: in dict key "x": cannot encode...')
38+
encode_error({1: 2}, 'dict has int key, want string')
39+
40+
## json.decode
41+
42+
assert.eq(json.decode("null"), None)
43+
assert.eq(json.decode("true"), True)
44+
assert.eq(json.decode("false"), False)
45+
assert.eq(json.decode("-123"), -123)
46+
assert.eq(json.decode("-0"), -0)
47+
assert.eq(json.decode("3539537889086624823140625"), 3539537889086624823140625)
48+
assert.eq(json.decode("3539537889086624823140625.0"), float(3539537889086624823140625))
49+
assert.eq(json.decode("3.539537889086625e+24"), 3.539537889086625e+24)
50+
assert.eq(json.decode("0e+1"), 0)
51+
assert.eq(json.decode("-0.0"), -0.0)
52+
assert.eq(json.decode(
53+
"-0.000000000000000000000000000000000000000000000000000000000000000000000000000001"),
54+
-0.000000000000000000000000000000000000000000000000000000000000000000000000000001)
55+
assert.eq(json.decode('[]'), [])
56+
assert.eq(json.decode('[1]'), [1])
57+
assert.eq(json.decode('[1,2,3]'), [1, 2, 3])
58+
assert.eq(json.decode('{"one": 1, "two": 2}'), dict(one=1, two=2))
59+
assert.eq(json.decode('{"foo\u0000bar": 42}'), {"foo\x00bar": 42})
60+
assert.eq(json.decode('"\ud83d\ude39\ud83d\udc8d"'), "😹💍")
61+
assert.eq(json.decode('"\u0123"'), 'ģ')
62+
assert.eq(json.decode('"\x7f"'), "\x7f")
63+
64+
def decode_error(expr, error):
65+
assert.fails(lambda: json.decode(expr), error)
66+
67+
decode_error('truefalse',
68+
"json.decode: at offset 4, unexpected character 'f' after value")
69+
70+
decode_error('"abc', "unclosed string literal")
71+
decode_error('"ab\gc"', "invalid character 'g' in string escape code")
72+
decode_error("'abc'", "unexpected character '\\\\''")
73+
74+
decode_error("1.2.3", "invalid number: 1.2.3")
75+
decode_error("+1", "unexpected character '\\+'")
76+
decode_error("-abc", "invalid number: -")
77+
decode_error("-", "invalid number: -")
78+
decode_error("-00", "invalid number: -00")
79+
decode_error("00", "invalid number: 00")
80+
decode_error("--1", "invalid number: --1")
81+
decode_error("-+1", "invalid number: -\\+1")
82+
decode_error("1e1e1", "invalid number: 1e1e1")
83+
decode_error("0123", "invalid number: 0123")
84+
decode_error("000.123", "invalid number: 000.123")
85+
decode_error("-0123", "invalid number: -0123")
86+
decode_error("-000.123", "invalid number: -000.123")
87+
decode_error("0x123", "unexpected character 'x' after value")
88+
89+
decode_error('[1, 2 ', "unexpected end of file")
90+
decode_error('[1, 2, ', "unexpected end of file")
91+
decode_error('[1, 2, ]', "unexpected character ']'")
92+
decode_error('[1, 2, }', "unexpected character '}'")
93+
decode_error('[1, 2}', "got '}', want ',' or ']'")
94+
95+
decode_error('{"one": 1', "unexpected end of file")
96+
decode_error('{"one" 1', "after object key, got '1', want ':'")
97+
decode_error('{"one": 1 "two": 2', "in object, got '\"', want ',' or '}'")
98+
decode_error('{"one": 1,', "unexpected end of file")
99+
decode_error('{"one": 1, }', "unexpected character '}'")
100+
decode_error('{"one": 1]', "in object, got ']', want ',' or '}'")
101+
102+
def codec(x):
103+
return json.decode(json.encode(x))
104+
105+
# string round-tripping
106+
strings = [
107+
"😿", # U+1F63F CRYING_CAT_FACE
108+
"🐱‍👤", # CAT FACE + ZERO WIDTH JOINER + BUST IN SILHOUETTE
109+
]
110+
assert.eq(codec(strings), strings)
111+
112+
# codepoints is a string with every 16-bit code point.
113+
codepoints = ''.join(['%c' % c for c in range(65536)])
114+
assert.eq(codec(codepoints), codepoints)
115+
116+
# number round-tripping
117+
numbers = [
118+
0, 1, -1, +1, 1.23e45, -1.23e-45,
119+
3539537889086624823140625,
120+
float(3539537889086624823140625),
121+
]
122+
assert.eq(codec(numbers), numbers)
123+
124+
## json.indent
125+
126+
s = json.encode(dict(x = 1, y = ["one", "two"]))
127+
128+
assert.eq(json.indent(s), '''{
129+
"x": 1,
130+
"y": [
131+
"one",
132+
"two"
133+
]
134+
}''')
135+
136+
assert.eq(json.decode(json.indent(s)), {"x": 1, "y": ["one", "two"]})
137+
138+
assert.eq(json.indent(s, prefix='¶', indent='–––'), '''{
139+
¶–––"x": 1,
140+
¶–––"y": [
141+
¶––––––"one",
142+
¶––––––"two"
143+
¶–––]
144+
¶}''')
145+
146+
assert.fails(lambda: json.indent("!@#$%^& this is not json"), 'invalid character')
147+
---

0 commit comments

Comments
 (0)