Skip to content

Commit ecfd33e

Browse files
kddnewtonkddnewton
committed
Add a test suite
1 parent f601a3d commit ecfd33e

File tree

7 files changed

+170
-72
lines changed

7 files changed

+170
-72
lines changed

‎.github/workflows/main.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,14 @@ jobs:
1010
CI: true
1111
steps:
1212
- uses: actions/checkout@master
13+
with:
14+
submodules: recursive
15+
1316
- uses: ruby/setup-ruby@v1
1417
with:
1518
bundler-cache: true
1619
ruby-version: '3.1'
20+
1721
- name: Test
1822
run: bundle exec rake test
1923
automerge:

‎.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "test/JSONTestSuite"]
2+
path = test/JSONTestSuite
3+
url = [email protected]:nst/JSONTestSuite.git

‎Gemfile.lock

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ GEM
2424
prettier_print
2525

2626
PLATFORMS
27+
arm64-darwin-21
2728
x86_64-darwin-21
2829
x86_64-linux
2930

‎lib/syntax_tree/json/ast.rb

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ def child_nodes
8787
[]
8888
end
8989

90-
alias deconstruct value
90+
alias deconstruct child_nodes
9191

9292
def deconstruct_keys(keys)
9393
{ value: value }
@@ -110,7 +110,7 @@ def child_nodes
110110
values.values
111111
end
112112

113-
alias deconstruct values
113+
alias deconstruct child_nodes
114114

115115
def deconstruct_keys(keys)
116116
{ values: values }
@@ -133,7 +133,7 @@ def child_nodes
133133
[value]
134134
end
135135

136-
alias deconstruct value
136+
alias deconstruct child_nodes
137137

138138
def deconstruct_keys(keys)
139139
{ value: value }
@@ -156,7 +156,7 @@ def child_nodes
156156
[]
157157
end
158158

159-
alias deconstruct value
159+
alias deconstruct child_nodes
160160

161161
def deconstruct_keys(keys)
162162
{ value: value }

‎lib/syntax_tree/json/parser.rb

Lines changed: 41 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,21 @@ module JSON
55
# This class is responsible for converting from a plain string input into
66
# an AST.
77
class Parser
8+
class ParseError < StandardError
9+
end
10+
811
attr_reader :source
912

1013
def initialize(source)
1114
@source = source
1215
end
1316

1417
def parse
15-
parse_item(make_tokens) => [value, []]
16-
AST::Root.new(value: value)
18+
if parse_item(make_tokens) in [value, []]
19+
AST::Root.new(value: value)
20+
else
21+
raise ParseError, "unexpected tokens after value"
22+
end
1723
end
1824

1925
private
@@ -33,7 +39,9 @@ def deconstruct_keys(keys)
3339
end
3440

3541
def make_tokens
36-
buffer = source.dup
42+
buffer = source.dup.force_encoding("UTF-8")
43+
raise ParseError, "invalid UTF-8" unless buffer.valid_encoding?
44+
3745
tokens = []
3846

3947
until buffer.empty?
@@ -43,14 +51,16 @@ def make_tokens
4351
Token.new(type: $&.to_sym)
4452
in /\A-?(0|[1-9]\d*)(\.\d+)?([Ee][-+]?\d+)?/
4553
Token.new(type: :number, value: $&)
46-
in /\A"[^"]*?"/
54+
in /\A"[^"\\]*(?:\\.[^"\\]*)*"/
4755
Token.new(type: :string, value: $&)
4856
in /\Atrue/
4957
Token.new(type: :true)
5058
in /\Afalse/
5159
Token.new(type: :false)
5260
in /\Anull/
5361
Token.new(type: :null)
62+
else
63+
raise ParseError, "unexpected token: #{buffer.strip[0]}"
5464
end
5565

5666
buffer = $'
@@ -71,6 +81,8 @@ def parse_array(tokens)
7181
return AST::Array.new(values: values), rest
7282
in [Token[type: :","], *rest]
7383
tokens = rest
84+
else
85+
raise ParseError, "expected ',' or ']' after array value"
7486
end
7587
end
7688
end
@@ -79,39 +91,46 @@ def parse_object(tokens)
7991
values = {}
8092

8193
loop do
82-
tokens => [Token[type: :string, value: key], Token[type: :":"], *tokens]
83-
value, tokens = parse_item(tokens)
84-
values[key] = value
85-
86-
case tokens
87-
in [Token[type: :"}"], *rest]
88-
return AST::Object.new(values: values), rest
89-
in [Token[type: :","], *rest]
90-
tokens = rest
94+
if tokens in [{ type: :string, value: key }, { type: :":" }, *tokens]
95+
value, tokens = parse_item(tokens)
96+
values[key] = value
97+
98+
case tokens
99+
in [{ type: :"}" }, *rest]
100+
return AST::Object.new(values: values), rest
101+
in [{ type: :"," }, *rest]
102+
tokens = rest
103+
else
104+
raise ParseError, "expected ',' or '}' after object value"
105+
end
106+
else
107+
raise ParseError, "expected key and ':' after opening '{'"
91108
end
92109
end
93110
end
94111

95112
def parse_item(tokens)
96113
case tokens
97-
in [Token[type: :"["], Token[type: :"]"], *rest]
114+
in [{ type: :"[" }, { type: :"]" }, *rest]
98115
[AST::Array.new(values: []), rest]
99-
in [Token[type: :"["], *rest]
116+
in [{ type: :"[" }, *rest]
100117
parse_array(rest)
101-
in [Token[type: :"{"], Token[type: :"}"], *rest]
118+
in [{ type: :"{" }, { type: :"}" }, *rest]
102119
[AST::Object.new(values: {}), rest]
103-
in [Token[type: :"{"], *rest]
120+
in [{ type: :"{" }, *rest]
104121
parse_object(rest)
105-
in [Token[type: :false], *rest]
122+
in [{ type: :false }, *rest]
106123
[AST::False.new, rest]
107-
in [Token[type: :true], *rest]
124+
in [{ type: :true }, *rest]
108125
[AST::True.new, rest]
109-
in [Token[type: :null], *rest]
126+
in [{ type: :null }, *rest]
110127
[AST::Null.new, rest]
111-
in [Token[type: :string, value: value], *rest]
128+
in [{ type: :string, value: value }, *rest]
112129
[AST::String.new(value: value), rest]
113-
in [Token[type: :number, value:], *rest]
130+
in [{ type: :number, value: }, *rest]
114131
[AST::Number.new(value: value), rest]
132+
else
133+
raise ParseError, "unexpected token: #{tokens.first&.type}"
115134
end
116135
end
117136
end

‎test/JSONTestSuite

Submodule JSONTestSuite added at d64aefb

‎test/json_test.rb

Lines changed: 116 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -5,56 +5,126 @@
55
# Here to make sure the thread-local variable gets set up correctly for 2.7.
66
PP.pp(nil, +"")
77

8-
class JSONTest < Minitest::Test
9-
def test_objects
10-
assert_format(<<~JSON)
11-
{
12-
"foo": "bar",
13-
"baz": "qux",
14-
"quux": "corge",
15-
"grault": "garply",
16-
"waldo": "fred",
17-
"plugh": "xyzzy",
18-
"thud": "thud"
19-
}
20-
JSON
21-
end
8+
module SyntaxTree
9+
class JSONTest < Minitest::Test
10+
def test_format
11+
assert_equal("true\n", JSON.format("true"))
12+
end
2213

23-
def test_arrays
24-
assert_format(<<~JSON)
25-
[
26-
"foo",
27-
"bar",
28-
"baz",
29-
"qux",
30-
"quux",
31-
"corge",
32-
"grault",
33-
"garply",
34-
"waldo",
35-
"fred",
36-
"plugh",
37-
"xyzzy",
38-
"thud",
39-
"thud"
40-
]
41-
JSON
42-
end
14+
def test_visitor
15+
JSON.parse("[true, false, null]").accept(JSON::Visitor.new)
16+
end
4317

44-
def test_literals
45-
assert_format("null\n")
46-
assert_format("true\n")
47-
assert_format("false\n")
48-
assert_format("\"foo\"\n")
49-
assert_format("1\n")
50-
end
18+
def test_objects
19+
assert_format(<<~JSON)
20+
{
21+
"foo": "bar",
22+
"baz": "qux",
23+
"quux": "corge",
24+
"grault": "garply",
25+
"waldo": "fred",
26+
"plugh": "xyzzy",
27+
"thud": "thud"
28+
}
29+
JSON
30+
end
31+
32+
def test_arrays
33+
assert_format(<<~JSON)
34+
[
35+
"foo",
36+
"bar",
37+
"baz",
38+
"qux",
39+
"quux",
40+
"corge",
41+
"grault",
42+
"garply",
43+
"waldo",
44+
"fred",
45+
"plugh",
46+
"xyzzy",
47+
"thud",
48+
"thud"
49+
]
50+
JSON
51+
end
52+
53+
def test_literals
54+
assert_format("null\n")
55+
assert_format("true\n")
56+
assert_format("false\n")
57+
assert_format("\"foo\"\n")
58+
assert_format("1\n")
59+
end
60+
61+
Dir["test/JSONTestSuite/test_parsing/y_*.json"].each do |filepath|
62+
define_method(:"test_#{filepath}") do
63+
parse(JSON.read(filepath))
64+
end
65+
end
66+
67+
KNOWN_N_FAILURES = %w[
68+
test/JSONTestSuite/test_parsing/n_multidigit_number_then_00.json
69+
test/JSONTestSuite/test_parsing/n_string_1_surrogate_then_escape_u.json
70+
test/JSONTestSuite/test_parsing/n_string_1_surrogate_then_escape_u1.json
71+
test/JSONTestSuite/test_parsing/n_string_1_surrogate_then_escape_u1x.json
72+
test/JSONTestSuite/test_parsing/n_string_1_surrogate_then_escape.json
73+
test/JSONTestSuite/test_parsing/n_string_backslash_00.json
74+
test/JSONTestSuite/test_parsing/n_string_escape_x.json
75+
test/JSONTestSuite/test_parsing/n_string_escaped_backslash_bad.json
76+
test/JSONTestSuite/test_parsing/n_string_escaped_ctrl_char_tab.json
77+
test/JSONTestSuite/test_parsing/n_string_escaped_emoji.json
78+
test/JSONTestSuite/test_parsing/n_string_incomplete_escape.json
79+
test/JSONTestSuite/test_parsing/n_string_incomplete_escaped_character.json
80+
test/JSONTestSuite/test_parsing/n_string_incomplete_surrogate_escape_invalid.json
81+
test/JSONTestSuite/test_parsing/n_string_incomplete_surrogate.json
82+
test/JSONTestSuite/test_parsing/n_string_invalid_backslash_esc.json
83+
test/JSONTestSuite/test_parsing/n_string_invalid_unicode_escape.json
84+
test/JSONTestSuite/test_parsing/n_string_unescaped_ctrl_char.json
85+
test/JSONTestSuite/test_parsing/n_string_unescaped_newline.json
86+
test/JSONTestSuite/test_parsing/n_string_unescaped_tab.json
87+
test/JSONTestSuite/test_parsing/n_string_unicode_CapitalU.json
88+
test/JSONTestSuite/test_parsing/n_structure_100000_opening_arrays.json
89+
test/JSONTestSuite/test_parsing/n_structure_null-byte-outside-string.json
90+
test/JSONTestSuite/test_parsing/n_structure_open_array_object.json
91+
test/JSONTestSuite/test_parsing/n_structure_whitespace_formfeed.json
92+
]
93+
94+
Dir["test/JSONTestSuite/test_parsing/n_*.json"].each do |filepath|
95+
define_method(:"test_#{filepath}") do
96+
skip if KNOWN_N_FAILURES.include?(filepath)
97+
98+
assert_raises(JSON::Parser::ParseError) do
99+
JSON.parse(JSON.read(filepath))
100+
end
101+
end
102+
end
103+
104+
private
105+
106+
def parse(source)
107+
# Test that is parses correctly
108+
parsed = JSON.parse(source)
109+
110+
# Test that it can be formatted and pretty-printed
111+
PrettierPrint.format(+"") { |q| parsed.format(q) }
112+
parsed.accept(JSON::PrettyPrint.new(PP.new(+"", 80)))
113+
114+
# Test that it can be pattern matched
115+
parsed in { value: { foo: :bar } }
116+
parsed in { value: [:foo, :bar] }
51117

52-
private
118+
# Return the tree so that it can be formatted
119+
parsed
120+
end
53121

54-
def assert_format(source)
55-
visitor = SyntaxTree::JSON::PrettyPrint.new(PP.new(+"", 80))
56-
SyntaxTree::JSON.parse(source).accept(visitor)
122+
def assert_format(source)
123+
parsed = parse(source)
124+
formatted =
125+
PrettierPrint.format(+"") { |q| parsed.accept(JSON::Format.new(q)) }
57126

58-
assert_equal(source, SyntaxTree::JSON.format(source))
127+
assert_equal(source, formatted)
128+
end
59129
end
60130
end

0 commit comments

Comments
 (0)