1//===-- ResourceScriptToken.cpp ---------------------------------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===---------------------------------------------------------------------===//
8//
9// This file implements an interface defined in ResourceScriptToken.h.
10// In particular, it defines an .rc script tokenizer.
11//
12//===---------------------------------------------------------------------===//
13
14#include "ResourceScriptToken.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/Support/raw_ostream.h"
17
18#include <cassert>
19#include <cctype>
20#include <cstdlib>
21#include <utility>
22
23using namespace llvm;
24
25using Kind = RCToken::Kind;
26
27// Checks if Representation is a correct description of an RC integer.
28// It should be a 32-bit unsigned integer, either decimal or hexadecimal
29// (0x[0-9a-f]+). For Windres mode, it can also be octal (0[0-7]+).
30// It might be followed by a single 'L' character (that is the difference
31// between our representation and StringRef's one). If Representation is
32// correct, 'true' is returned and the return value is put back in Num.
33static bool rcGetAsInteger(StringRef Representation, uint32_t &Num) {
34 size_t Length = Representation.size();
35 if (Length == 0)
36 return false;
37 // Strip the last 'L' if unnecessary.
38 if (std::toupper(c: Representation.back()) == 'L')
39 Representation = Representation.drop_back(N: 1);
40
41 return !Representation.getAsInteger<uint32_t>(Radix: 0, Result&: Num);
42}
43
44RCToken::RCToken(RCToken::Kind RCTokenKind, StringRef Value)
45 : TokenKind(RCTokenKind), TokenValue(Value) {}
46
47uint32_t RCToken::intValue() const {
48 assert(TokenKind == Kind::Int);
49 // We assume that the token already is a correct integer (checked by
50 // rcGetAsInteger).
51 uint32_t Result;
52 bool IsSuccess = rcGetAsInteger(Representation: TokenValue, Num&: Result);
53 assert(IsSuccess);
54 (void)IsSuccess; // Silence the compiler warning when -DNDEBUG flag is on.
55 return Result;
56}
57
58bool RCToken::isLongInt() const {
59 return TokenKind == Kind::Int && std::toupper(c: TokenValue.back()) == 'L';
60}
61
62StringRef RCToken::value() const { return TokenValue; }
63
64Kind RCToken::kind() const { return TokenKind; }
65
66bool RCToken::isLowPrecedenceBinaryOp() const {
67 switch (TokenKind) {
68 case Kind::Plus:
69 case Kind::Minus:
70 case Kind::Pipe:
71 case Kind::Amp:
72 return true;
73 default:
74 return false;
75 }
76}
77
78bool RCToken::isHighPrecedenceBinaryOp() const {
79 switch (TokenKind) {
80 case Kind::Asterisk:
81 case Kind::Slash:
82 return true;
83 default:
84 return false;
85 }
86}
87
88static Error getStringError(const Twine &message) {
89 return make_error<StringError>(Args: "Error parsing file: " + message,
90 Args: inconvertibleErrorCode());
91}
92
93namespace {
94
95class Tokenizer {
96public:
97 Tokenizer(StringRef Input, bool IsWindres)
98 : Data(Input), DataLength(Input.size()), Pos(0), IsWindres(IsWindres) {}
99
100 Expected<std::vector<RCToken>> run();
101
102private:
103 // All 'advancing' methods return boolean values; if they're equal to false,
104 // the stream has ended or failed.
105 bool advance(size_t Amount = 1);
106 bool skipWhitespaces();
107
108 // Consumes a token. If any problem occurred, a non-empty Error is returned.
109 Error consumeToken(const Kind TokenKind);
110
111 // Check if tokenizer is about to read FollowingChars.
112 bool willNowRead(StringRef FollowingChars) const;
113
114 // Check if tokenizer can start reading an identifier at current position.
115 // The original tool did non specify the rules to determine what is a correct
116 // identifier. We assume they should follow the C convention:
117 // [a-zA-Z_][a-zA-Z0-9_]*.
118 bool canStartIdentifier() const;
119 // Check if tokenizer can continue reading an identifier.
120 bool canContinueIdentifier() const;
121
122 // Check if tokenizer can start reading an integer.
123 // A correct integer always starts with a 0-9 digit,
124 // can contain characters 0-9A-Fa-f (digits),
125 // Ll (marking the integer is 32-bit), Xx (marking the representation
126 // is hexadecimal). As some kind of separator should come after the
127 // integer, we can consume the integer until a non-alphanumeric
128 // character.
129 bool canStartInt() const;
130 bool canContinueInt() const;
131 void trimIntString(StringRef &Str) const;
132
133 bool canStartString() const;
134
135 // Check if tokenizer can start reading a single line comment (e.g. a comment
136 // that begins with '//')
137 bool canStartLineComment() const;
138
139 // Check if tokenizer can start or finish reading a block comment (e.g. a
140 // comment that begins with '/*' and ends with '*/')
141 bool canStartBlockComment() const;
142
143 // Throw away all remaining characters on the current line.
144 void skipCurrentLine();
145
146 bool streamEof() const;
147
148 // Classify the token that is about to be read from the current position.
149 Kind classifyCurrentToken() const;
150
151 // Process the Kind::Identifier token - check if it is
152 // an identifier describing a block start or end.
153 void processIdentifier(RCToken &token) const;
154
155 StringRef Data;
156 size_t DataLength, Pos;
157 bool IsWindres;
158};
159
160void Tokenizer::skipCurrentLine() {
161 Pos = Data.find_first_of(Chars: "\r\n", From: Pos);
162 Pos = Data.find_first_not_of(Chars: "\r\n", From: Pos);
163
164 if (Pos == StringRef::npos)
165 Pos = DataLength;
166}
167
168Expected<std::vector<RCToken>> Tokenizer::run() {
169 Pos = 0;
170 std::vector<RCToken> Result;
171
172 // Consume an optional UTF-8 Byte Order Mark.
173 if (willNowRead(FollowingChars: "\xef\xbb\xbf"))
174 advance(Amount: 3);
175
176 while (!streamEof()) {
177 if (!skipWhitespaces())
178 break;
179
180 Kind TokenKind = classifyCurrentToken();
181 if (TokenKind == Kind::Invalid)
182 return getStringError(message: "Invalid token found at position " + Twine(Pos));
183
184 const size_t TokenStart = Pos;
185 if (Error TokenError = consumeToken(TokenKind))
186 return std::move(TokenError);
187
188 // Comments are just deleted, don't bother saving them.
189 if (TokenKind == Kind::LineComment || TokenKind == Kind::StartComment)
190 continue;
191
192 StringRef Contents = Data.take_front(N: Pos).drop_front(N: TokenStart);
193
194 if (TokenKind == Kind::Int)
195 trimIntString(Str&: Contents);
196
197 RCToken Token(TokenKind, Contents);
198 if (TokenKind == Kind::Identifier) {
199 processIdentifier(token&: Token);
200 } else if (TokenKind == Kind::Int) {
201 uint32_t TokenInt;
202 if (!rcGetAsInteger(Representation: Token.value(), Num&: TokenInt)) {
203 // The integer has incorrect format or cannot be represented in
204 // a 32-bit integer.
205 return getStringError(message: "Integer invalid or too large: " +
206 Token.value().str());
207 }
208 }
209
210 Result.push_back(x: Token);
211 }
212
213 return Result;
214}
215
216bool Tokenizer::advance(size_t Amount) {
217 Pos += Amount;
218 return !streamEof();
219}
220
221bool Tokenizer::skipWhitespaces() {
222 while (!streamEof() && isSpace(C: Data[Pos]))
223 advance();
224 return !streamEof();
225}
226
227Error Tokenizer::consumeToken(const Kind TokenKind) {
228 switch (TokenKind) {
229 // One-character token consumption.
230#define TOKEN(Name)
231#define SHORT_TOKEN(Name, Ch) case Kind::Name:
232#include "ResourceScriptTokenList.def"
233 advance();
234 return Error::success();
235
236 case Kind::LineComment:
237 advance(Amount: 2);
238 skipCurrentLine();
239 return Error::success();
240
241 case Kind::StartComment: {
242 advance(Amount: 2);
243 auto EndPos = Data.find(Str: "*/", From: Pos);
244 if (EndPos == StringRef::npos)
245 return getStringError(
246 message: "Unclosed multi-line comment beginning at position " + Twine(Pos));
247 advance(Amount: EndPos - Pos);
248 advance(Amount: 2);
249 return Error::success();
250 }
251 case Kind::Identifier:
252 while (!streamEof() && canContinueIdentifier())
253 advance();
254 return Error::success();
255
256 case Kind::Int:
257 while (!streamEof() && canContinueInt())
258 advance();
259 return Error::success();
260
261 case Kind::String:
262 // Consume the preceding 'L', if there is any.
263 if (std::toupper(c: Data[Pos]) == 'L')
264 advance();
265 // Consume the double-quote.
266 advance();
267
268 // Consume the characters until the end of the file, line or string.
269 while (true) {
270 if (streamEof()) {
271 return getStringError(message: "Unterminated string literal.");
272 } else if (Data[Pos] == '"') {
273 // Consume the ending double-quote.
274 advance();
275 // However, if another '"' follows this double-quote, the string didn't
276 // end and we just included '"' into the string.
277 if (!willNowRead(FollowingChars: "\""))
278 return Error::success();
279 } else if (Data[Pos] == '\n') {
280 return getStringError(message: "String literal not terminated in the line.");
281 }
282
283 advance();
284 }
285
286 case Kind::Invalid:
287 assert(false && "Cannot consume an invalid token.");
288 }
289
290 llvm_unreachable("Unknown RCToken::Kind");
291}
292
293bool Tokenizer::willNowRead(StringRef FollowingChars) const {
294 return Data.drop_front(N: Pos).starts_with(Prefix: FollowingChars);
295}
296
297bool Tokenizer::canStartIdentifier() const {
298 assert(!streamEof());
299
300 const char CurChar = Data[Pos];
301 return std::isalpha(CurChar) || CurChar == '_' || CurChar == '.';
302}
303
304bool Tokenizer::canContinueIdentifier() const {
305 assert(!streamEof());
306 const char CurChar = Data[Pos];
307 return std::isalnum(CurChar) || CurChar == '_' || CurChar == '.' ||
308 CurChar == '/' || CurChar == '\\' || CurChar == '-';
309}
310
311bool Tokenizer::canStartInt() const {
312 assert(!streamEof());
313 return std::isdigit(Data[Pos]);
314}
315
316bool Tokenizer::canStartBlockComment() const {
317 assert(!streamEof());
318 return Data.drop_front(N: Pos).starts_with(Prefix: "/*");
319}
320
321bool Tokenizer::canStartLineComment() const {
322 assert(!streamEof());
323 return Data.drop_front(N: Pos).starts_with(Prefix: "//");
324}
325
326bool Tokenizer::canContinueInt() const {
327 assert(!streamEof());
328 return std::isalnum(Data[Pos]);
329}
330
331bool Tokenizer::canStartString() const {
332 return willNowRead(FollowingChars: "\"") || willNowRead(FollowingChars: "L\"") || willNowRead(FollowingChars: "l\"");
333}
334
335bool Tokenizer::streamEof() const { return Pos == DataLength; }
336
337Kind Tokenizer::classifyCurrentToken() const {
338 if (canStartBlockComment())
339 return Kind::StartComment;
340 if (canStartLineComment())
341 return Kind::LineComment;
342
343 if (canStartInt())
344 return Kind::Int;
345 if (canStartString())
346 return Kind::String;
347 // BEGIN and END are at this point of lexing recognized as identifiers.
348 if (canStartIdentifier())
349 return Kind::Identifier;
350
351 const char CurChar = Data[Pos];
352
353 switch (CurChar) {
354 // One-character token classification.
355#define TOKEN(Name)
356#define SHORT_TOKEN(Name, Ch) \
357 case Ch: \
358 return Kind::Name;
359#include "ResourceScriptTokenList.def"
360
361 default:
362 return Kind::Invalid;
363 }
364}
365
366void Tokenizer::processIdentifier(RCToken &Token) const {
367 assert(Token.kind() == Kind::Identifier);
368 StringRef Name = Token.value();
369
370 if (Name.equals_insensitive(RHS: "begin"))
371 Token = RCToken(Kind::BlockBegin, Name);
372 else if (Name.equals_insensitive(RHS: "end"))
373 Token = RCToken(Kind::BlockEnd, Name);
374}
375
376void Tokenizer::trimIntString(StringRef &Str) const {
377 if (!IsWindres) {
378 // For compatibility with rc.exe, strip leading zeros that make the
379 // integer literal interpreted as octal.
380 //
381 // We do rely on Stringref::getAsInteger for autodetecting between
382 // decimal and hexadecimal literals, but we want to avoid interpreting
383 // literals as octal.
384 //
385 // This omits the leading zeros from the RCToken's value string entirely,
386 // which also has a visible effect when dumping the tokenizer output.
387 // Alternatively, we could store the IsWindres flag in RCToken and defer
388 // the trimming to RCToken::intValue.
389 while (Str.size() >= 2 && Str[0] == '0' && std::isdigit(Str[1]))
390 Str = Str.drop_front(N: 1);
391 }
392}
393
394} // anonymous namespace
395
396namespace llvm {
397
398Expected<std::vector<RCToken>> tokenizeRC(StringRef Input, bool IsWindres) {
399 return Tokenizer(Input, IsWindres).run();
400}
401
402} // namespace llvm
403