1//===--- UnicodeCharSetsGenerator.cpp - Unicode character sets generator --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is used to generate clang/lib/Lex/UnicodeCharSetsGenerated.cpp and
10// llvm/lib/Support/UnicodeCharSetsGenerated.cpp from the XML representation of
11// the Unicode Character Database available at
12// https://www.unicode.org/Public/UCD/latest/ucdxml/
13//
14//===----------------------------------------------------------------------===//
15
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Support/CommandLine.h"
20#include "llvm/Support/FileSystem.h"
21#include "llvm/Support/FormatVariadic.h"
22#include "llvm/Support/InitLLVM.h"
23#include "llvm/Support/ToolOutputFile.h"
24#include "llvm/Support/UnicodeCharRanges.h"
25#include "llvm/Support/WithColor.h"
26
27#include <libxml/xmlreader.h>
28#include <vector>
29
30using namespace llvm;
31
32static cl::opt<std::string> UCDFile(cl::Positional, cl::Required,
33 cl::desc("<UCD XML file>"));
34static cl::opt<std::string> ClangOutput(cl::Positional, cl::Required,
35 cl::desc("<clang output file>"));
36static cl::opt<std::string> LLVMOutput(cl::Positional, cl::Required,
37 cl::desc("<LLVM output file>"));
38
39static constexpr unsigned CodePointCount = 0x110000;
40
41static constexpr StringLiteral FileHeader = R"(
42//===----------------------------------------------------------------------===//
43//
44// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
45// See https://llvm.org/LICENSE.txt for license information.
46// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
47//
48//===----------------------------------------------------------------------===//
49//
50// This file was generated by llvm/utils/UnicodeData/UnicodeCharSetsGenerator
51// from the Unicode {0} Character Database. Do not edit manually.
52//
53//===----------------------------------------------------------------------===//
54
55#include "llvm/Support/UnicodeCharRanges.h"
56
57using llvm::sys::UnicodeCharRange;
58using CharRanges = llvm::sys::UnicodeCharSet::CharRanges;
59
60namespace {1} {{
61)";
62
63[[noreturn]] static void error(const Twine &Message) {
64 WithColor::error() << Message << '\n';
65 exit(EXIT_FAILURE);
66}
67
68namespace {
69struct Properties {
70 unsigned XIDStart : 1;
71 unsigned XIDContinue : 1;
72 unsigned MathStart : 1;
73 unsigned MathContinue : 1;
74 unsigned Char : 1;
75 unsigned Printable : 1;
76 unsigned Control : 1;
77 unsigned Format : 1;
78 unsigned Combining : 1;
79 unsigned DoubleWidth : 1;
80};
81
82struct CharacterDatabase {
83 std::string Version;
84 std::vector<Properties> Chars;
85};
86
87struct XmlDeleter {
88 void operator()(xmlTextReader *Reader) { xmlFreeTextReader(reader: Reader); }
89 void operator()(xmlChar *Ptr) { xmlFree(Ptr); }
90};
91} // namespace
92
93static StringRef fromXmlChar(const xmlChar *S) {
94 return S ? StringRef(reinterpret_cast<const char *>(S)) : StringRef();
95}
96
97static bool isPrintableGC(StringRef GC) {
98 return GC == "Zs" || (!GC.empty() && StringRef("LMNPS").contains(C: GC.front()));
99}
100
101// Loads the properties we care about from the flat XML representation of the
102// UCD, which describes each code point, or range of code points, with all of
103// its properties, e.g.
104// <char cp="0041" age="1.1" na="LATIN CAPITAL LETTER A" gc="Lu" ... />
105static CharacterDatabase loadDatabase(StringRef Path) {
106 std::unique_ptr<xmlTextReader, XmlDeleter> Reader(xmlReaderForFile(
107 filename: Path.str().c_str(), /*encoding=*/nullptr, options: XML_PARSE_NONET));
108 if (!Reader)
109 error(Message: Path + ": cannot open file");
110
111 CharacterDatabase Database;
112 Database.Chars.resize(new_size: CodePointCount);
113 int Status;
114 while ((Status = xmlTextReaderRead(reader: Reader.get())) == 1) {
115 if (xmlTextReaderNodeType(reader: Reader.get()) != XML_READER_TYPE_ELEMENT)
116 continue;
117 StringRef Element = fromXmlChar(S: xmlTextReaderConstLocalName(reader: Reader.get()));
118 if (Element == "description") {
119 // <description>Unicode 18.0.0</description>
120 std::unique_ptr<xmlChar, XmlDeleter> Description(
121 xmlTextReaderReadString(reader: Reader.get()));
122 if (Description)
123 Database.Version = fromXmlChar(S: Description.get())
124 .rsplit(Separator: ' ')
125 .second.rsplit(Separator: '.')
126 .first.str();
127 continue;
128 }
129 if (!is_contained(Set: {"char", "reserved", "noncharacter", "surrogate"},
130 Element))
131 continue;
132
133 unsigned First = 0, Last = 0;
134 bool HasRange = false;
135 unsigned Found = 0;
136 Properties P = {};
137 while (xmlTextReaderMoveToNextAttribute(reader: Reader.get()) == 1) {
138 StringRef Name = fromXmlChar(S: xmlTextReaderConstLocalName(reader: Reader.get()));
139 StringRef Value = fromXmlChar(S: xmlTextReaderConstValue(reader: Reader.get()));
140 P.Char = Element == "char";
141 if (Name == "cp" || Name == "first-cp") {
142 if (Value.getAsInteger(Radix: 16, Result&: First))
143 break;
144 if (Name == "cp")
145 Last = First;
146 HasRange = true;
147 } else if (Name == "last-cp") {
148 if (Value.getAsInteger(Radix: 16, Result&: Last))
149 HasRange = false;
150 } else if (Name == "XIDS") {
151 P.XIDStart = Value == "Y";
152 ++Found;
153 } else if (Name == "XIDC") {
154 P.XIDContinue = Value == "Y";
155 ++Found;
156 } else if (Name == "ID_Compat_Math_Start") {
157 P.MathStart = Value == "Y";
158 ++Found;
159 } else if (Name == "ID_Compat_Math_Continue") {
160 P.MathContinue = Value == "Y";
161 ++Found;
162 } else if (Name == "gc") {
163 P.Printable = isPrintableGC(GC: Value);
164 P.Control = Value == "Cc";
165 P.Format = Value == "Cf";
166 P.Combining = Value == "Mn" || Value == "Me";
167 ++Found;
168 } else if (Name == "ea") {
169 P.DoubleWidth = Value == "F" || Value == "W";
170 ++Found;
171 }
172 }
173
174 if (!HasRange || First > Last || Last >= CodePointCount || Found != 6)
175 error(Message: Path + ":" + Twine(xmlTextReaderGetParserLineNumber(reader: Reader.get())) +
176 ": invalid <" + Element + "> element");
177 llvm::fill(Range: MutableArrayRef(Database.Chars).slice(N: First, M: Last - First + 1),
178 Value&: P);
179 }
180 if (Status != 0)
181 error(Message: Path + ": failed to parse XML");
182 if (Database.Version.empty())
183 error(Message: Path + ": missing Unicode version");
184 return Database;
185}
186
187template <typename Predicate>
188static void emitTable(raw_ostream &OS, StringRef Name, StringRef Description,
189 ArrayRef<Properties> Chars, Predicate Pred) {
190 SmallVector<sys::UnicodeCharRange> Ranges;
191 for (unsigned CodePoint = 0, E = Chars.size(); CodePoint != E; ++CodePoint) {
192 if (!Pred(Chars[CodePoint]))
193 continue;
194 if (!Ranges.empty() && Ranges.back().Upper + 1 == CodePoint)
195 Ranges.back().Upper = CodePoint;
196 else
197 Ranges.push_back(Elt: {.Lower: CodePoint, .Upper: CodePoint});
198 }
199
200 OS << "\n// " << Description << "\nstatic constexpr UnicodeCharRange " << Name
201 << "Data[] = {\n";
202 for (sys::UnicodeCharRange Range : Ranges)
203 OS << formatv(Fmt: " {{{0:X4}, {1:X4}},\n", Vals&: Range.Lower, Vals&: Range.Upper);
204 OS << "};\nextern const CharRanges " << Name << " = " << Name << "Data;\n";
205}
206
207static void writeFile(StringRef Path, StringRef Namespace, StringRef Version,
208 function_ref<void(raw_ostream &)> EmitTables) {
209 std::error_code EC;
210 ToolOutputFile Out(Path, EC, sys::fs::OF_Text);
211 if (EC)
212 error(Message: Path + ": " + EC.message());
213
214 Out.os() << formatv(Fmt: FileHeader.drop_front().data(), Vals&: Version, Vals&: Namespace);
215 EmitTables(Out.os());
216 Out.os() << "\n} // namespace " << Namespace << '\n';
217 Out.keep();
218}
219
220int main(int argc, char **argv) {
221 InitLLVM X(argc, argv);
222 cl::ParseCommandLineOptions(
223 argc, argv,
224 Overview: "Unicode character sets generator\n\n"
225 " Generates clang/lib/Lex/UnicodeCharSetsGenerated.cpp and\n"
226 " llvm/lib/Support/UnicodeCharSetsGenerated.cpp from the flat XML\n"
227 " representation of the Unicode Character Database, which can be\n"
228 " downloaded from https://www.unicode.org/Public/UCD/latest/ucdxml/\n"
229 " The generated files must then be formatted with clang-format.\n");
230
231 CharacterDatabase UCD = loadDatabase(Path: UCDFile);
232 ArrayRef<Properties> Chars = UCD.Chars;
233
234 writeFile(Path: ClangOutput, Namespace: "clang", Version: UCD.Version, EmitTables: [&](raw_ostream &OS) {
235 emitTable(OS, Name: "GeneratedXIDStartRanges", Description: "XID_Start", Chars,
236 Pred: [](Properties P) { return P.XIDStart; });
237 emitTable(OS, Name: "GeneratedXIDContinueRanges",
238 Description: "XID_Continue, excluding XID_Start", Chars,
239 Pred: [](Properties P) { return P.XIDContinue && !P.XIDStart; });
240 emitTable(OS, Name: "GeneratedMathematicalNotationProfileIDStartRanges",
241 Description: "ID_Compat_Math_Start", Chars,
242 Pred: [](Properties P) { return P.MathStart; });
243 emitTable(OS, Name: "GeneratedMathematicalNotationProfileIDContinueRanges",
244 Description: "ID_Compat_Math_Continue, excluding ID_Compat_Math_Start", Chars,
245 Pred: [](Properties P) { return P.MathContinue && !P.MathStart; });
246 });
247 writeFile(
248 Path: LLVMOutput, Namespace: "llvm::sys::unicode", Version: UCD.Version, EmitTables: [&](raw_ostream &OS) {
249 emitTable(OS, Name: "GeneratedCharacterRanges", Description: "Characters", Chars,
250 Pred: [](Properties P) { return P.Char; });
251 emitTable(OS, Name: "GeneratedPrintableRanges",
252 Description: "General_Category=L|M|N|P|S|Zs", Chars,
253 Pred: [](Properties P) { return P.Printable; });
254 emitTable(OS, Name: "GeneratedControlCharacterRanges", Description: "General_Category=Cc",
255 Chars, Pred: [](Properties P) { return P.Control; });
256 emitTable(OS, Name: "GeneratedFormatCharacterRanges", Description: "General_Category=Cf",
257 Chars, Pred: [](Properties P) { return P.Format; });
258 emitTable(OS, Name: "GeneratedCombiningCharacterRanges",
259 Description: "General_Category=Mn|Me", Chars,
260 Pred: [](Properties P) { return P.Combining; });
261 emitTable(OS, Name: "GeneratedDoubleWidthCharacterRanges",
262 Description: "East_Asian_Width=F|W", Chars,
263 Pred: [](Properties P) { return P.DoubleWidth; });
264 });
265}
266