1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Simple drivers to test the mustache spec found at:
10// https://github.com/mustache/spec
11//
12// It is used to verify that the current implementation conforms to the spec.
13// Simply download the spec and pass the test JSON files to the driver. Each
14// spec file should have a list of tests for compliance with the spec. These
15// are loaded as test cases, and rendered with our Mustache implementation,
16// which is then compared against the expected output from the spec.
17//
18// The current implementation only supports non-optional parts of the spec, so
19// we do not expect any of the dynamic-names, inheritance, or lambda tests to
20// pass. Additionally, Triple Mustache is not supported. Unsupported tests are
21// marked as XFail and are removed from the XFail list as they are fixed.
22//
23// Usage:
24// llvm-test-mustache-spec path/to/test/file.json path/to/test/file2.json ...
25//===----------------------------------------------------------------------===//
26
27#include "llvm/ADT/StringSet.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/Debug.h"
30#include "llvm/Support/Error.h"
31#include "llvm/Support/MemoryBuffer.h"
32#include "llvm/Support/Mustache.h"
33#include "llvm/Support/Path.h"
34#include <string>
35
36using namespace llvm;
37using namespace llvm::json;
38using namespace llvm::mustache;
39
40#define DEBUG_TYPE "llvm-test-mustache-spec"
41
42static cl::OptionCategory Cat("llvm-test-mustache-spec Options");
43
44static cl::list<std::string>
45 InputFiles(cl::Positional, cl::desc("<input files>"), cl::OneOrMore);
46
47static cl::opt<bool> ReportErrors("report-errors",
48 cl::desc("Report errors in spec tests"),
49 cl::cat(Cat));
50
51static ExitOnError ExitOnErr;
52
53static int NumXFail = 0;
54static int NumSuccess = 0;
55
56static const StringMap<StringSet<>> XFailTestNames = {{
57 {"delimiters.json",
58 {
59 "Pair Behavior",
60 "Special Characters",
61 "Sections",
62 "Inverted Sections",
63 "Partial Inheritence",
64 "Post-Partial Behavior",
65 "Standalone Tag",
66 "Indented Standalone Tag",
67 "Standalone Line Endings",
68 "Standalone Without Previous Line",
69 "Standalone Without Newline",
70 }},
71 {"~dynamic-names.json",
72 {
73 "Basic Behavior - Partial",
74 "Basic Behavior - Name Resolution",
75 "Context",
76 "Dotted Names",
77 "Dotted Names - Failed Lookup",
78 "Dotted names - Context Stacking",
79 "Dotted names - Context Stacking Under Repetition",
80 "Dotted names - Context Stacking Failed Lookup",
81 "Recursion",
82 "Surrounding Whitespace",
83 "Inline Indentation",
84 "Standalone Line Endings",
85 "Standalone Without Previous Line",
86 "Standalone Without Newline",
87 "Standalone Indentation",
88 "Padding Whitespace",
89 }},
90 {"~inheritance.json",
91 {
92 "Default",
93 "Variable",
94 "Triple Mustache",
95 "Sections",
96 "Negative Sections",
97 "Mustache Injection",
98 "Inherit",
99 "Overridden content",
100 "Data does not override block default",
101 "Two overridden parents",
102 "Override parent with newlines",
103 "Inherit indentation",
104 "Only one override",
105 "Parent template",
106 "Recursion",
107 "Multi-level inheritance, no sub child",
108 "Text inside parent",
109 "Text inside parent",
110 "Block scope",
111 "Standalone parent",
112 "Standalone block",
113 "Block reindentation",
114 "Intrinsic indentation",
115 "Nested block reindentation",
116
117 }},
118 {"~lambdas.json",
119 {
120 "Interpolation",
121 "Interpolation - Expansion",
122 "Interpolation - Alternate Delimiters",
123 "Interpolation - Multiple Calls",
124 "Escaping",
125 "Section",
126 "Section - Expansion",
127 "Section - Alternate Delimiters",
128 "Section - Multiple Calls",
129
130 }},
131 {"interpolation.json",
132 {
133 "Triple Mustache",
134 "Triple Mustache Integer Interpolation",
135 "Triple Mustache Decimal Interpolation",
136 "Triple Mustache Null Interpolation",
137 "Triple Mustache Context Miss Interpolation",
138 "Dotted Names - Triple Mustache Interpolation",
139 "Implicit Iterators - Triple Mustache",
140 "Triple Mustache - Surrounding Whitespace",
141 "Triple Mustache - Standalone",
142 "Triple Mustache With Padding",
143 }},
144 {"partials.json", {"Standalone Indentation"}},
145 {"sections.json", {"Implicit Iterator - Triple mustache"}},
146}};
147
148struct TestData {
149 TestData() = default;
150 explicit TestData(const json::Object &TestCase)
151 : TemplateStr(*TestCase.getString(K: "template")),
152 ExpectedStr(*TestCase.getString(K: "expected")),
153 Name(*TestCase.getString(K: "name")), Data(TestCase.get(K: "data")),
154 Partials(TestCase.get(K: "partials")) {}
155
156 static Expected<TestData> createTestData(json::Object *TestCase,
157 StringRef InputFile) {
158 // If any of the needed elements are missing, we cannot continue.
159 // NOTE: partials are optional in the test schema.
160 if (!TestCase || !TestCase->getString(K: "template") ||
161 !TestCase->getString(K: "expected") || !TestCase->getString(K: "name") ||
162 !TestCase->get(K: "data"))
163 return createStringError(
164 EC: llvm::inconvertibleErrorCode(),
165 S: "invalid JSON schema in test file: " + InputFile + "\n");
166
167 return TestData(*TestCase);
168 }
169
170 StringRef TemplateStr;
171 StringRef ExpectedStr;
172 StringRef Name;
173 const Value *Data;
174 const Value *Partials;
175};
176
177static void reportTestFailure(const TestData &TD, StringRef ActualStr,
178 bool IsXFail) {
179 LLVM_DEBUG(dbgs() << "Template: " << TD.TemplateStr << "\n");
180 if (TD.Partials) {
181 LLVM_DEBUG(dbgs() << "Partial: ");
182 LLVM_DEBUG(TD.Partials->print(dbgs()));
183 LLVM_DEBUG(dbgs() << "\n");
184 }
185 LLVM_DEBUG(dbgs() << "JSON Data: ");
186 LLVM_DEBUG(TD.Data->print(dbgs()));
187 LLVM_DEBUG(dbgs() << "\n");
188 outs() << formatv(Fmt: "Test {}: {}\n", Vals: (IsXFail ? "XFailed" : "Failed"), Vals: TD.Name);
189 if (ReportErrors) {
190 outs() << " Expected: \'" << TD.ExpectedStr << "\'\n"
191 << " Actual: \'" << ActualStr << "\'\n"
192 << " ====================\n";
193 }
194}
195
196static void registerPartials(const Value *Partials, Template &T) {
197 if (!Partials)
198 return;
199 for (const auto &[Partial, Str] : *Partials->getAsObject())
200 T.registerPartial(Name: Partial.str(), Partial: Str.getAsString()->str());
201}
202
203static json::Value readJsonFromFile(StringRef &InputFile) {
204 std::unique_ptr<MemoryBuffer> Buffer =
205 ExitOnErr(errorOrToExpected(EO: MemoryBuffer::getFile(Filename: InputFile)));
206 return ExitOnErr(parse(JSON: Buffer->getBuffer()));
207}
208
209static bool isTestXFail(StringRef FileName, StringRef TestName) {
210 auto P = llvm::sys::path::filename(path: FileName);
211 auto It = XFailTestNames.find(Key: P);
212 return It != XFailTestNames.end() && It->second.contains(key: TestName);
213}
214
215static bool evaluateTest(StringRef &InputFile, TestData &TestData,
216 std::string &ActualStr) {
217 bool IsXFail = isTestXFail(FileName: InputFile, TestName: TestData.Name);
218 bool Matches = TestData.ExpectedStr == ActualStr;
219 if ((Matches && IsXFail) || (!Matches && !IsXFail)) {
220 reportTestFailure(TD: TestData, ActualStr, IsXFail);
221 return false;
222 }
223 IsXFail ? NumXFail++ : NumSuccess++;
224 return true;
225}
226
227static void runTest(StringRef InputFile) {
228 NumXFail = 0;
229 NumSuccess = 0;
230 outs() << "Running Tests: " << InputFile << "\n";
231 json::Value Json = readJsonFromFile(InputFile);
232
233 json::Object *Obj = Json.getAsObject();
234 Array *TestArray = Obj->getArray(K: "tests");
235 // Even though we parsed the JSON, it can have a bad format, so check it.
236 if (!TestArray)
237 ExitOnErr(createStringError(
238 EC: llvm::inconvertibleErrorCode(),
239 S: "invalid JSON schema in test file: " + InputFile + "\n"));
240
241 const size_t Total = TestArray->size();
242
243 for (Value V : *TestArray) {
244 auto TestData =
245 ExitOnErr(TestData::createTestData(TestCase: V.getAsObject(), InputFile));
246 Template T(TestData.TemplateStr);
247 registerPartials(Partials: TestData.Partials, T);
248
249 std::string ActualStr;
250 raw_string_ostream OS(ActualStr);
251 T.render(Data: *TestData.Data, OS);
252 evaluateTest(InputFile, TestData, ActualStr);
253 }
254
255 const int NumFailed = Total - NumSuccess - NumXFail;
256 outs() << formatv(Fmt: "===Results===\n"
257 " Suceeded: {}\n"
258 " Expectedly Failed: {}\n"
259 " Failed: {}\n"
260 " Total: {}\n",
261 Vals&: NumSuccess, Vals&: NumXFail, Vals: NumFailed, Vals: Total);
262}
263
264int main(int argc, char **argv) {
265 ExitOnErr.setBanner(std::string(argv[0]) + " error: ");
266 cl::ParseCommandLineOptions(argc, argv);
267 for (const auto &FileName : InputFiles)
268 runTest(InputFile: FileName);
269 return 0;
270}
271