hanzoai/postcss
2
0
Fork 0
mirror of https://github.com/hanzoai/postcss.git synced 2026-09-24 17:01:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make tokenizer really streamed

Browse Source
This commit is contained in:
Andrey Sitnik
2017-05-01 12:17:55 +02:00
parent 5389969c86
commit 6d9e508246
5 changed files with 277 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/parse.es6
-1
View File
@@ -10,7 +10,6 @@ export default function parse(css, opts) {
let input = new Input(css, opts);
let parser = new Parser(input);
try {
parser.tokenize();
parser.parse();
} catch (e) {
if ( e.name === 'CssSyntaxError' && opts && opts.from ) {
lib/parser.es6
+29 -39
View File
@@ -10,23 +10,20 @@ export default class Parser {
constructor(input) {
this.input = input;
this.pos = 0;
this.root = new Root();
this.current = this.root;
this.spaces = '';
this.semicolon = false;
this.root.source = { input, start: { line: 1, column: 1 } };
}
this.tokenizer = tokenizer(this.input);
tokenize() {
this.tokens = tokenizer(this.input).tokenize();
this.root.source = { input, start: { line: 1, column: 1 } };
}
parse() {
let token;
while ( this.pos < this.tokens.length ) {
token = this.tokens[this.pos];
while ( !this.tokenizer.endOfFile() ) {
token = this.tokenizer.nextToken();
switch ( token[0] ) {
@@ -52,11 +49,9 @@ export default class Parser {
break;
default:
this.other();
this.other(token);
break;
}
this.pos += 1;
}
this.endFile();
}
@@ -87,18 +82,18 @@ export default class Parser {
this.current = node;
}
other() {
let token;
other(start) {
let end = false;
let type = null;
let colon = false;
let bracket = null;
let brackets = [];
let start = this.pos;
while ( this.pos < this.tokens.length ) {
token = this.tokens[this.pos];
type = token[0];
let tokens = [];
let token = start;
while ( token ) {
type = token[0];
tokens.push(token);
if ( type === '(' || type === '[' ) {
if ( !bracket ) bracket = token;
@@ -107,18 +102,18 @@ export default class Parser {
} else if ( brackets.length === 0 ) {
if ( type === ';' ) {
if ( colon ) {
this.decl(this.tokens.slice(start, this.pos + 1));
this.decl(tokens);
return;
} else {
break;
}
} else if ( type === '{' ) {
this.rule(this.tokens.slice(start, this.pos + 1));
this.rule(tokens);
return;
} else if ( type === '}' ) {
this.pos -= 1;
this.tokenizer.back(tokens.pop());
end = true;
break;
@@ -131,26 +126,23 @@ export default class Parser {
if ( brackets.length === 0 ) bracket = null;
}
this.pos += 1;
}
if ( this.pos === this.tokens.length ) {
this.pos -= 1;
end = true;
token = this.tokenizer.nextToken();
}
if ( this.tokenizer.endOfFile() ) end = true;
if ( brackets.length > 0 ) this.unclosedBracket(bracket);
if ( end && colon ) {
while ( this.pos > start ) {
token = this.tokens[this.pos][0];
while ( tokens.length ) {
token = tokens[tokens.length - 1][0];
if ( token !== 'space' && token !== 'comment' ) break;
this.pos -= 1;
this.tokenizer.back(tokens.pop());
}
this.decl(this.tokens.slice(start, this.pos + 1));
this.decl(tokens);
return;
} else {
this.unknownWord(start);
}
this.unknownWord(start);
}
rule(tokens) {
@@ -262,9 +254,8 @@ export default class Parser {
let open = false;
let params = [];
this.pos += 1;
while ( this.pos < this.tokens.length ) {
token = this.tokens[this.pos];
while ( !this.tokenizer.endOfFile() ) {
token = this.tokenizer.nextToken();
if ( token[0] === ';' ) {
node.source.end = { line: token[2], column: token[3] };
@@ -280,10 +271,10 @@ export default class Parser {
params.push(token);
}
this.pos += 1;
}
if ( this.pos === this.tokens.length ) {
last = true;
if ( this.tokenizer.endOfFile() ) {
last = true;
break;
}
}
node.raws.between = this.spacesAndCommentsFromEnd(params);
@@ -439,8 +430,7 @@ export default class Parser {
throw this.input.error('Unclosed bracket', bracket[2], bracket[3]);
}
unknownWord(start) {
let token = this.tokens[start];
unknownWord(token) {
throw this.input.error('Unknown word', token[2], token[3]);
}
lib/terminal-highlight.es6
+17 -13
View File
@@ -23,36 +23,40 @@ const HIGHLIGHT_THEME = {
';': colors.yellow
};
function getTokenType([type, value], index, tokens) {
if (type === 'word') {
if (value[0] === '.') {
function getTokenType([type, value], processor) {
if ( type === 'word' ) {
if ( value[0] === '.' ) {
return 'class';
}
if (value[0] === '#') {
if ( value[0] === '#' ) {
return 'hash';
}
}
let nextToken = tokens[index + 1];
if (nextToken && (nextToken[0] === 'brackets' || nextToken[0] === '(')) {
return 'call';
if ( !processor.endOfFile() ) {
let next = processor.nextToken();
processor.back(next);
if ( next[0] === 'brackets' || next[0] === '(' ) return 'call';
}
return type;
}
function terminalHighlight(css) {
let tokens = tokenizer(new Input(css), { ignoreErrors: true }).tokenize();
return tokens.map((token, index) => {
let color = HIGHLIGHT_THEME[getTokenType(token, index, tokens)];
let processor = tokenizer(new Input(css), { ignoreErrors: true });
let result = '';
while ( !processor.endOfFile() ) {
let token = processor.nextToken();
let color = HIGHLIGHT_THEME[getTokenType(token, processor)];
if ( color ) {
return token[1].split(/\r?\n/)
result += token[1].split(/\r?\n/)
.map( i => color(i) )
.join('\n');
} else {
return token[1];
result += token[1];
}
}).join('');
}
return result;
}
export default terminalHighlight;
lib/tokenize.es6
+211 -221
View File
@@ -35,144 +35,92 @@ export default function tokenizer(input, options = {}) {
let line = 1;
let pos = 0;
let buffer = [];
let returned = [];
function unclosed(what) {
throw input.error('Unclosed ' + what, line, pos - offset);
}
function endOfFile() {
return pos >= length;
return returned.length === 0 && pos >= length;
}
function lastSymbol() {
return pos === length;
}
function nextToken() {
if ( returned.length ) return returned.pop();
if ( pos >= length ) return;
function readNextToken() {
if (!endOfFile()) {
code = css.charCodeAt(pos);
code = css.charCodeAt(pos);
if (code === NEWLINE || code === FEED ||
code === CR && css.charCodeAt(pos + 1) !== NEWLINE) {
offset = pos;
line += 1;
}
if (code === NEWLINE || code === FEED ||
code === CR && css.charCodeAt(pos + 1) !== NEWLINE) {
offset = pos;
line += 1;
}
switch (code) {
case NEWLINE:
case SPACE:
case TAB:
case CR:
case FEED:
next = pos;
do {
next += 1;
code = css.charCodeAt(next);
if (code === NEWLINE) {
offset = next;
line += 1;
}
} while ( code === SPACE ||
code === NEWLINE ||
code === TAB ||
code === CR ||
code === FEED );
switch (code) {
case NEWLINE:
case SPACE:
case TAB:
case CR:
case FEED:
currentToken = ['space', css.slice(pos, next)];
pos = next - 1;
break;
case OPEN_SQUARE:
currentToken = ['[', '[', line, pos - offset];
break;
case CLOSE_SQUARE:
currentToken = [']', ']', line, pos - offset];
break;
case OPEN_CURLY:
currentToken = ['{', '{', line, pos - offset];
break;
case CLOSE_CURLY:
currentToken = ['}', '}', line, pos - offset];
break;
case COLON:
currentToken = [':', ':', line, pos - offset];
break;
case SEMICOLON:
currentToken = [';', ';', line, pos - offset];
break;
case OPEN_PARENTHESES:
prev = buffer.length ? buffer.pop()[1] : '';
n = css.charCodeAt(pos + 1);
if ( prev === 'url' &&
n !== SINGLE_QUOTE && n !== DOUBLE_QUOTE &&
n !== SPACE && n !== NEWLINE && n !== TAB &&
n !== FEED && n !== CR ) {
next = pos;
do {
next += 1;
code = css.charCodeAt(next);
if (code === NEWLINE) {
offset = next;
line += 1;
}
} while ( code === SPACE ||
code === NEWLINE ||
code === TAB ||
code === CR ||
code === FEED );
currentToken = ['space', css.slice(pos, next)];
pos = next - 1;
break;
case OPEN_SQUARE:
currentToken = ['[', '[', line, pos - offset];
break;
case CLOSE_SQUARE:
currentToken = [']', ']', line, pos - offset];
break;
case OPEN_CURLY:
currentToken = ['{', '{', line, pos - offset];
break;
case CLOSE_CURLY:
currentToken = ['}', '}', line, pos - offset];
break;
case COLON:
currentToken = [':', ':', line, pos - offset];
break;
case SEMICOLON:
currentToken = [';', ';', line, pos - offset];
break;
case OPEN_PARENTHESES:
prev = buffer.length ? buffer.pop()[1] : '';
n = css.charCodeAt(pos + 1);
if ( prev === 'url' &&
n !== SINGLE_QUOTE && n !== DOUBLE_QUOTE &&
n !== SPACE && n !== NEWLINE && n !== TAB &&
n !== FEED && n !== CR ) {
next = pos;
do {
escaped = false;
next = css.indexOf(')', next + 1);
if ( next === -1 ) {
if ( ignore ) {
next = pos;
break;
} else {
unclosed('bracket');
}
}
escapePos = next;
while ( css.charCodeAt(escapePos - 1) === BACKSLASH ) {
escapePos -= 1;
escaped = !escaped;
}
} while ( escaped );
currentToken = ['brackets', css.slice(pos, next + 1),
line, pos - offset,
line, next - offset
];
pos = next;
} else {
next = css.indexOf(')', pos + 1);
content = css.slice(pos, next + 1);
if ( next === -1 || RE_BAD_BRACKET.test(content) ) {
currentToken = ['(', '(', line, pos - offset];
} else {
currentToken = ['brackets', content,
line, pos - offset,
line, next - offset
];
pos = next;
}
}
break;
case CLOSE_PARENTHESES:
currentToken = [')', ')', line, pos - offset];
break;
case SINGLE_QUOTE:
case DOUBLE_QUOTE:
quote = code === SINGLE_QUOTE ? '\'' : '"';
next = pos;
do {
escaped = false;
next = css.indexOf(quote, next + 1);
next = css.indexOf(')', next + 1);
if ( next === -1 ) {
if ( ignore ) {
next = pos + 1;
next = pos;
break;
} else {
unclosed('string');
unclosed('bracket');
}
}
escapePos = next;
@@ -182,6 +130,131 @@ export default function tokenizer(input, options = {}) {
}
} while ( escaped );
currentToken = ['brackets', css.slice(pos, next + 1),
line, pos - offset,
line, next - offset
];
pos = next;
} else {
next = css.indexOf(')', pos + 1);
content = css.slice(pos, next + 1);
if ( next === -1 || RE_BAD_BRACKET.test(content) ) {
currentToken = ['(', '(', line, pos - offset];
} else {
currentToken = ['brackets', content,
line, pos - offset,
line, next - offset
];
pos = next;
}
}
break;
case CLOSE_PARENTHESES:
currentToken = [')', ')', line, pos - offset];
break;
case SINGLE_QUOTE:
case DOUBLE_QUOTE:
quote = code === SINGLE_QUOTE ? '\'' : '"';
next = pos;
do {
escaped = false;
next = css.indexOf(quote, next + 1);
if ( next === -1 ) {
if ( ignore ) {
next = pos + 1;
break;
} else {
unclosed('string');
}
}
escapePos = next;
while ( css.charCodeAt(escapePos - 1) === BACKSLASH ) {
escapePos -= 1;
escaped = !escaped;
}
} while ( escaped );
content = css.slice(pos, next + 1);
lines = content.split('\n');
last = lines.length - 1;
if ( last > 0 ) {
nextLine = line + last;
nextOffset = next - lines[last].length;
} else {
nextLine = line;
nextOffset = offset;
}
currentToken = ['string', css.slice(pos, next + 1),
line, pos - offset,
nextLine, next - nextOffset
];
offset = nextOffset;
line = nextLine;
pos = next;
break;
case AT:
RE_AT_END.lastIndex = pos + 1;
RE_AT_END.test(css);
if ( RE_AT_END.lastIndex === 0 ) {
next = css.length - 1;
} else {
next = RE_AT_END.lastIndex - 2;
}
currentToken = ['at-word', css.slice(pos, next + 1),
line, pos - offset,
line, next - offset
];
pos = next;
break;
case BACKSLASH:
next = pos;
escape = true;
while ( css.charCodeAt(next + 1) === BACKSLASH ) {
next += 1;
escape = !escape;
}
code = css.charCodeAt(next + 1);
if ( escape && (code !== SLASH &&
code !== SPACE &&
code !== NEWLINE &&
code !== TAB &&
code !== CR &&
code !== FEED ) ) {
next += 1;
}
currentToken = ['word', css.slice(pos, next + 1),
line, pos - offset,
line, next - offset
];
pos = next;
break;
default:
if ( code === SLASH && css.charCodeAt(pos + 1) === ASTERISK ) {
next = css.indexOf('*/', pos + 2) + 1;
if ( next === 0 ) {
if ( ignore ) {
next = css.length;
} else {
unclosed('comment');
}
}
content = css.slice(pos, next + 1);
lines = content.split('\n');
last = lines.length - 1;
@@ -194,48 +267,22 @@ export default function tokenizer(input, options = {}) {
nextOffset = offset;
}
currentToken = ['string', css.slice(pos, next + 1),
line, pos - offset,
currentToken = ['comment', content,
line, pos - offset,
nextLine, next - nextOffset
];
offset = nextOffset;
line = nextLine;
pos = next;
break;
case AT:
RE_AT_END.lastIndex = pos + 1;
RE_AT_END.test(css);
if ( RE_AT_END.lastIndex === 0 ) {
} else {
RE_WORD_END.lastIndex = pos + 1;
RE_WORD_END.test(css);
if ( RE_WORD_END.lastIndex === 0 ) {
next = css.length - 1;
} else {
next = RE_AT_END.lastIndex - 2;
}
currentToken = ['at-word', css.slice(pos, next + 1),
line, pos - offset,
line, next - offset
];
pos = next;
break;
case BACKSLASH:
next = pos;
escape = true;
while ( css.charCodeAt(next + 1) === BACKSLASH ) {
next += 1;
escape = !escape;
}
code = css.charCodeAt(next + 1);
if ( escape && (code !== SLASH &&
code !== SPACE &&
code !== NEWLINE &&
code !== TAB &&
code !== CR &&
code !== FEED ) ) {
next += 1;
next = RE_WORD_END.lastIndex - 2;
}
currentToken = ['word', css.slice(pos, next + 1),
@@ -243,82 +290,25 @@ export default function tokenizer(input, options = {}) {
line, next - offset
];
buffer.push(currentToken);
pos = next;
break;
default:
if ( code === SLASH && css.charCodeAt(pos + 1) === ASTERISK ) {
next = css.indexOf('*/', pos + 2) + 1;
if ( next === 0 ) {
if ( ignore ) {
next = css.length;
} else {
unclosed('comment');
}
}
content = css.slice(pos, next + 1);
lines = content.split('\n');
last = lines.length - 1;
if ( last > 0 ) {
nextLine = line + last;
nextOffset = next - lines[last].length;
} else {
nextLine = line;
nextOffset = offset;
}
currentToken = ['comment', content,
line, pos - offset,
nextLine, next - nextOffset
];
offset = nextOffset;
line = nextLine;
pos = next;
} else {
RE_WORD_END.lastIndex = pos + 1;
RE_WORD_END.test(css);
if ( RE_WORD_END.lastIndex === 0 ) {
next = css.length - 1;
} else {
next = RE_WORD_END.lastIndex - 2;
}
currentToken = ['word', css.slice(pos, next + 1),
line, pos - offset,
line, next - offset
];
buffer.push(currentToken);
pos = next;
}
break;
}
pos++;
return currentToken;
break;
}
pos++;
return currentToken;
}
/* Tokenizer driver */
function tokenize() {
let tokens = [];
while (!endOfFile()) {
tokens.push(readNextToken());
}
return tokens;
function back(token) {
returned.push(token);
}
return {
readNextToken,
endOfFile,
tokenize,
lastSymbol
back,
nextToken,
endOfFile
};
}
test/tokenize.js
+20 -13
View File
@@ -3,14 +3,17 @@ import Input from '../lib/input';
import test from 'ava';
function run(t, css, opts, tokens) {
if ( typeof tokens === 'undefined' ) [tokens, opts] = [opts, tokens];
t.deepEqual(tokenizer(new Input(css, opts)).tokenize(), tokens);
function tokenize(css, opts) {
let processor = tokenizer(new Input(css), opts);
let tokens = [];
while (!processor.endOfFile()) {
tokens.push(processor.nextToken());
}
return tokens;
}
function ignoreRun(t, css, tokens) {
let tk = tokenizer(new Input(css), { ignoreErrors: true });
t.deepEqual(tk.tokenize(), tokens);
function run(t, css, tokens, opts) {
t.deepEqual(tokenize(css, opts), tokens);
}
test('tokenizes empty file', t => {
@@ -221,33 +224,37 @@ test('tokenizes CSS', t => {
test('throws error on unclosed string', t => {
t.throws(() => {
tokenizer(new Input(' "')).tokenize();
tokenize(' "');
}, /:1:2: Unclosed string/);
});
test('throws error on unclosed comment', t => {
t.throws(() => {
tokenizer(new Input(' /*')).tokenize();
tokenize(' /*');
}, /:1:2: Unclosed comment/);
});
test('throws error on unclosed url', t => {
t.throws(() => {
tokenizer(new Input('url(')).tokenize();
tokenize('url(');
}, /:1:4: Unclosed bracket/);
});
test('ignores unclosing string on request', t => {
ignoreRun(t, ' "', [['space', ' '], ['string', '\"', 1, 2, 1, 3]]);
run(t, ' "', [
['space', ' '], ['string', '\"', 1, 2, 1, 3]
], { ignoreErrors: true });
});
test('ignores unclosing comment on request', t => {
ignoreRun(t, ' /*', [['space', ' '], ['comment', '/*', 1, 2, 1, 4]]);
run(t, ' /*', [
['space', ' '], ['comment', '/*', 1, 2, 1, 4]
], { ignoreErrors: true });
});
test('ignores unclosing comment on request', t => {
ignoreRun(t, 'url(', [
run(t, 'url(', [
['word', 'url', 1, 1, 1, 3],
['brackets', '(', 1, 4, 1, 4]
]);
], { ignoreErrors: true });
});