281 lines
8.3 KiB
JavaScript
281 lines
8.3 KiB
JavaScript
var charCodeDef = require('./char-code-definitions');
|
|
var isDigit = charCodeDef.isDigit;
|
|
var isHexDigit = charCodeDef.isHexDigit;
|
|
var isUppercaseLetter = charCodeDef.isUppercaseLetter;
|
|
var isLowercaseLetter = charCodeDef.isLowercaseLetter;
|
|
var isLetter = charCodeDef.isLetter;
|
|
var isNonAscii = charCodeDef.isNonAscii;
|
|
var isNameStart = charCodeDef.isNameStart;
|
|
var isName = charCodeDef.isName;
|
|
var isNonPrintable = charCodeDef.isNonPrintable;
|
|
var isNewline = charCodeDef.isNewline;
|
|
var isWhiteSpace = charCodeDef.isWhiteSpace;
|
|
var isValidEscape = charCodeDef.isValidEscape;
|
|
var isIdentifierStart = charCodeDef.isIdentifierStart;
|
|
var isNumberStart = charCodeDef.isNumberStart;
|
|
|
|
// detect BOM (https://en.wikipedia.org/wiki/Byte_order_mark)
|
|
function firstCharOffset(source) {
|
|
if (source.length > 0) {
|
|
if (source.charCodeAt(0) === 0xFEFF || // UTF-16BE
|
|
source.charCodeAt(0) === 0xFFFE) { // UTF-16LE
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
function getCharCode(source, offset) {
|
|
return offset < source.length ? source.charCodeAt(offset) : 0;
|
|
}
|
|
|
|
function getNewlineLength(source, offset, code) {
|
|
if (code === 13 /* \r */ && getCharCode(source, offset + 1) === 10 /* \n */) {
|
|
return 2;
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
function cmpChar(testStr, offset, referenceCode) {
|
|
var code = testStr.charCodeAt(offset);
|
|
|
|
// code.toLowerCase() for A..Z
|
|
if (isUppercaseLetter(code)) {
|
|
code = code | 32;
|
|
}
|
|
|
|
return code === referenceCode;
|
|
}
|
|
|
|
function cmpStr(testStr, start, end, referenceStr) {
|
|
if (end - start !== referenceStr.length) {
|
|
return false;
|
|
}
|
|
|
|
if (start < 0 || end > testStr.length) {
|
|
return false;
|
|
}
|
|
|
|
for (var i = start; i < end; i++) {
|
|
var testCode = testStr.charCodeAt(i);
|
|
var referenceCode = referenceStr.charCodeAt(i - start);
|
|
|
|
// testCode.toLowerCase() for A..Z
|
|
if (isUppercaseLetter(testCode)) {
|
|
testCode = testCode | 32;
|
|
}
|
|
|
|
if (testCode !== referenceCode) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
function findWhiteSpaceStart(source, offset) {
|
|
for (; offset >= 0; offset--) {
|
|
if (!isWhiteSpace(source.charCodeAt(offset))) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
return offset + 1;
|
|
}
|
|
|
|
function findWhiteSpaceEnd(source, offset) {
|
|
for (; offset < source.length; offset++) {
|
|
if (!isWhiteSpace(source.charCodeAt(offset))) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
return offset;
|
|
}
|
|
|
|
function findDecimalNumberEnd(source, offset) {
|
|
for (; offset < source.length; offset++) {
|
|
if (!isDigit(source.charCodeAt(offset))) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
return offset;
|
|
}
|
|
|
|
// § 4.3.7. Consume an escaped code point
|
|
function consumeEscaped(source, offset) {
|
|
// It assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed and
|
|
// that the next input code point has already been verified to be part of a valid escape.
|
|
offset += 2;
|
|
|
|
// hex digit
|
|
if (isHexDigit(getCharCode(source, offset - 1))) {
|
|
// Consume as many hex digits as possible, but no more than 5.
|
|
// Note that this means 1-6 hex digits have been consumed in total.
|
|
for (var maxOffset = Math.min(source.length, offset + 5); offset < maxOffset; offset++) {
|
|
if (!isHexDigit(getCharCode(source, offset))) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
// If the next input code point is whitespace, consume it as well.
|
|
var code = getCharCode(source, offset);
|
|
if (isWhiteSpace(code)) {
|
|
offset += getNewlineLength(source, offset, code);
|
|
}
|
|
}
|
|
|
|
return offset;
|
|
}
|
|
|
|
// §4.3.11. Consume a name
|
|
// Note: This algorithm does not do the verification of the first few code points that are necessary
|
|
// to ensure the returned code points would constitute an <ident-token>. If that is the intended use,
|
|
// ensure that the stream starts with an identifier before calling this algorithm.
|
|
function consumeName(source, offset) {
|
|
// Let result initially be an empty string.
|
|
// Repeatedly consume the next input code point from the stream:
|
|
for (; offset < source.length; offset++) {
|
|
var code = source.charCodeAt(offset);
|
|
|
|
// name code point
|
|
if (isName(code)) {
|
|
// Append the code point to result.
|
|
continue;
|
|
}
|
|
|
|
// the stream starts with a valid escape
|
|
if (isValidEscape(code, getCharCode(source, offset + 1))) {
|
|
// Consume an escaped code point. Append the returned code point to result.
|
|
offset = consumeEscaped(source, offset) - 1;
|
|
continue;
|
|
}
|
|
|
|
// anything else
|
|
// Reconsume the current input code point. Return result.
|
|
break;
|
|
}
|
|
|
|
return offset;
|
|
}
|
|
|
|
// §4.3.12. Consume a number
|
|
function consumeNumber(source, offset) {
|
|
var code = source.charCodeAt(offset);
|
|
|
|
// 2. If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-),
|
|
// consume it and append it to repr.
|
|
if (code === 0x002B || code === 0x002D) {
|
|
code = source.charCodeAt(offset += 1);
|
|
}
|
|
|
|
// 3. While the next input code point is a digit, consume it and append it to repr.
|
|
if (isDigit(code)) {
|
|
offset = findDecimalNumberEnd(source, offset + 1);
|
|
code = source.charCodeAt(offset);
|
|
}
|
|
|
|
// 4. If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then:
|
|
if (code === 0x002E && isDigit(source.charCodeAt(offset + 1))) {
|
|
// 4.1 Consume them.
|
|
// 4.2 Append them to repr.
|
|
code = source.charCodeAt(offset += 2);
|
|
|
|
// 4.3 Set type to "number".
|
|
// TODO
|
|
|
|
// 4.4 While the next input code point is a digit, consume it and append it to repr.
|
|
|
|
offset = findDecimalNumberEnd(source, offset);
|
|
}
|
|
|
|
// 5. If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E)
|
|
// or U+0065 LATIN SMALL LETTER E (e), ... , followed by a digit, then:
|
|
if (cmpChar(source, offset, 101 /* e */)) {
|
|
var sign = 0;
|
|
code = source.charCodeAt(offset + 1);
|
|
|
|
// ... optionally followed by U+002D HYPHEN-MINUS (-) or U+002B PLUS SIGN (+) ...
|
|
if (code === 0x002D || code === 0x002B) {
|
|
sign = 1;
|
|
code = source.charCodeAt(offset + 2);
|
|
}
|
|
|
|
// ... followed by a digit
|
|
if (isDigit(code)) {
|
|
// 5.1 Consume them.
|
|
// 5.2 Append them to repr.
|
|
|
|
// 5.3 Set type to "number".
|
|
// TODO
|
|
|
|
// 5.4 While the next input code point is a digit, consume it and append it to repr.
|
|
offset = findDecimalNumberEnd(source, offset + 1 + sign + 1);
|
|
}
|
|
}
|
|
|
|
return offset;
|
|
}
|
|
|
|
// § 4.3.14. Consume the remnants of a bad url
|
|
// ... its sole use is to consume enough of the input stream to reach a recovery point
|
|
// where normal tokenizing can resume.
|
|
function consumeBadUrlRemnants(source, offset) {
|
|
// Repeatedly consume the next input code point from the stream:
|
|
for (; offset < source.length; offset++) {
|
|
var code = source.charCodeAt(offset);
|
|
|
|
// U+0029 RIGHT PARENTHESIS ())
|
|
// EOF
|
|
if (code === 0x0029) {
|
|
// Return.
|
|
offset++;
|
|
break;
|
|
}
|
|
|
|
if (isValidEscape(code, getCharCode(source, offset + 1))) {
|
|
// Consume an escaped code point.
|
|
// Note: This allows an escaped right parenthesis ("\)") to be encountered
|
|
// without ending the <bad-url-token>. This is otherwise identical to
|
|
// the "anything else" clause.
|
|
offset = consumeEscaped(source, offset);
|
|
}
|
|
}
|
|
|
|
return offset;
|
|
}
|
|
|
|
module.exports = {
|
|
firstCharOffset: firstCharOffset,
|
|
|
|
isDigit: isDigit,
|
|
isHexDigit: isHexDigit,
|
|
isUppercaseLetter: isUppercaseLetter,
|
|
isLowercaseLetter: isLowercaseLetter,
|
|
isLetter: isLetter,
|
|
isNonAscii: isNonAscii,
|
|
isNameStart: isNameStart,
|
|
isName: isName,
|
|
isNonPrintable: isNonPrintable,
|
|
isNewline: isNewline,
|
|
isWhiteSpace: isWhiteSpace,
|
|
isValidEscape: isValidEscape,
|
|
isIdentifierStart: isIdentifierStart,
|
|
isNumberStart: isNumberStart,
|
|
|
|
consumeEscaped: consumeEscaped,
|
|
consumeName: consumeName,
|
|
consumeNumber: consumeNumber,
|
|
consumeBadUrlRemnants: consumeBadUrlRemnants,
|
|
|
|
cmpChar: cmpChar,
|
|
cmpStr: cmpStr,
|
|
|
|
getNewlineLength: getNewlineLength,
|
|
findWhiteSpaceStart: findWhiteSpaceStart,
|
|
findWhiteSpaceEnd: findWhiteSpaceEnd
|
|
};
|