wikimedia/mediawiki-extensions-VisualEditor
1
0
Fork 0
mirror of https://gerrit.wikimedia.org/r/mediawiki/extensions/VisualEditor synced 2024-11-24 22:35:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clean up newline handling. Emit a NEWLINE token for each

Browse source 
non-{comment,pre,nowiki} newline.
This commit is contained in:
Gabriel Wicke 2011-12-08 14:34:18 +00:00
parent abc2254110
commit c2b69e2486
1 changed files with 26 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
modules/parser/pegTokenizer.pegjs.txt
View file

@ -401,7 +401,8 @@
start start
= e:toplevelblock* newline* { = e:toplevelblock* newline* {
return flatten(e); // XXX: move doQuotes out to general token stream transformer
return doQuotes(flatten(e));
} }
@ -505,6 +506,8 @@ eof = & { return isEOF(pos); } { return true; }
newline newline
= '\n' / '\r\n' = '\n' / '\r\n'
newlineToken = newline { return [{type: 'NEWLINE'}] }
eolf = newline / eof eolf = newline / eof
toplevelblock toplevelblock
@ -517,8 +520,8 @@ toplevelblock
} }
bs.attribs.push(['data-sourcePos', blockStart + ':' + pos]); bs.attribs.push(['data-sourcePos', blockStart + ':' + pos]);
// XXX: only run this for lines that actually need it! // XXX: only run this for lines that actually need it!
b.push({type: 'NEWLINE'}); //b.push({type: 'NEWLINE'});
b = doQuotes(b); // Move this to a token stream transform!
return b; return b;
} }
@ -1084,7 +1087,7 @@ list_char = [*#:;]
/* Tables */ /* Tables */
table table
= tas:table_start space* c:table_caption? b:table_body? table_end { = tas:table_start space* c:table_caption? b:table_body? te:table_end {
var res = {type: 'TAG', name: 'table'} var res = {type: 'TAG', name: 'table'}
var body = b !== '' ? b : []; var body = b !== '' ? b : [];
dp("body: " + pp(body)); dp("body: " + pp(body));
@ -1096,7 +1099,7 @@ table
if (c != '') { if (c != '') {
var caption = [{type: 'TAG', name: 'caption'}] var caption = [{type: 'TAG', name: 'caption'}]
.concat(c, [{type: 'ENDTAG', name: 'caption'}]); .concat(c, [{type: 'ENDTAG', name: 'caption'}], te);
} else { } else {
var caption = []; var caption = [];
} }
@ -1122,9 +1125,9 @@ table_attribs
= text / ! inline_breaks !newline ![|] c:. { return c } = text / ! inline_breaks !newline ![|] c:. { return c }
table_caption table_caption
= newline = n:newlineToken
"|+" c:inline* { "|+" c:inline* {
return c; return n.concat(c);
} }
table_body table_body
@ -1148,15 +1151,15 @@ table_firstrow
table_row table_row
= //& { dp("table row enter"); return true; } = //& { dp("table row enter"); return true; }
newline n:newlineToken
"|-" thtd_attribs? space* td:(table_data / table_header)* { "|-" thtd_attribs? space* td:(table_data / table_header)* {
return [{type: 'TAG', name: 'tr'}] return n.concat([{type: 'TAG', name: 'tr'}]
.concat(td, [{type: 'ENDTAG', name: 'tr'}]); , td, [{type: 'ENDTAG', name: 'tr'}]);
} }
table_data table_data
= //& { dp("table_data enter, pos=" + pos + input.substr(pos,10)); return true; } = //& { dp("table_data enter, pos=" + pos + input.substr(pos,10)); return true; }
("||" / newline "|") n:("||" { return [] } / nt:newlineToken "|" { return nt })
! [}+-] ! [}+-]
//& { dp('before attrib, pos=' + pos); return true; } //& { dp('before attrib, pos=' + pos); return true; }
a:(as:generic_attribute+ space* "|" !"|" { return as } )? a:(as:generic_attribute+ space* "|" !"|" { return as } )?
@ -1169,19 +1172,19 @@ table_data
a = []; a = [];
} }
//dp("table data result: " + pp(td) + ", attribts: " + pp(a)); //dp("table data result: " + pp(td) + ", attribts: " + pp(a));
return [{ type: 'TAG', name: 'td', attribs: a}] return n.concat( [{ type: 'TAG', name: 'td', attribs: a}]
.concat(td, [{type: 'ENDTAG', name: 'td'}]); , td, [{type: 'ENDTAG', name: 'td'}] );
} }
table_header table_header
= ("!!" / newline "!") = n:("!!" { return [] } / nl:newlineToken "!" { return nl })
a:(as:generic_attribute+ "!" !"!" { return as } )? a:(as:generic_attribute+ "!" !"!" { return as } )?
c:inline { c:inline {
if ( a == '' ) { if ( a == '' ) {
a = []; a = [];
} }
return [{type: 'TAG', name: 'th', attribs: a}] return n.concat( [{type: 'TAG', name: 'th', attribs: a}]
.concat(c, [{type: 'ENDTAG', name: 'th'}]); , c, [{type: 'ENDTAG', name: 'th'}]);
} }
thtd_attribs thtd_attribs
@ -1192,8 +1195,13 @@ thtd_attribs
table_end table_end
= newline? "|}" { clearFlag('table'); } = nt:newlineToken? ( "|}" / eof ) {
/ newline? eof clearFlag('table');
if(nt)
return nt;
else
return [];
}
/* Tabs do not mix well with the hybrid production syntax */ /* Tabs do not mix well with the hybrid production syntax */