mediawiki-extensions-Visual.../modules/parser/pegTokenizer.pegjs.txt

1269 lines
38 KiB
Plaintext
Raw Normal View History

/* Combined Wiki (MediaWiki) and HTML tokenizer. Produces a token stream
** (actually a list of tokens for now) suitable for a HTML5TreeBuilder. */
2011-11-02 21:09:19 +00:00
{
2011-11-30 13:42:26 +00:00
/* Fixme: use static functions to separate module! Unfortunately, this
* does not work:
* var tu = require('./mediawiki.tokenizer.utils.js');
* console.log(tu.flatten([]));
* Using exports in the module gets a bit further, but accesses to
* tu.flatten in productions still fail. Thus, I just moved the functions
* here until a solution is found:
*/
/* Static utilities */
// Flatten a list of lists.
var flatten = function ( e ) {
// Fast single-level flatten:
//return [].concat.apply([], e);
2011-11-30 13:42:26 +00:00
var es = [];
// flatten sub-arrays
for(var i = 0, length = e.length; i < length; i++) {
var ei = e[i];
if ($.isArray(ei))
es = es.concat(flatten(ei));
else
es.push(ei);
};
return es;
};
// Remove escaped quotes from attributes etc
2011-12-30 09:35:57 +00:00
// This was in the original PEG parser, but could not find anything in
// MediaWiki that supports \' and \"-style escaping. So remove? -- gwicke
2011-11-30 13:42:26 +00:00
var unquote = function (quotec, text) {
return text.replace('\\' + quotec, quotec);
};
// Decode html entities. In a browser, this should only be fed the entity,
// not untrusted html! XXX: replace with safer version.
var unentity = function ( entity ) {
return $("<div/>").html(entity).text();
};
2011-11-30 13:42:26 +00:00
// Debug print with global switch
2011-11-02 21:09:19 +00:00
var dp = function ( msg ) {
if ( false ) {
console.log(msg);
}
};
var pp = function ( s ) { return JSON.stringify(s, null, 2); }
2011-11-30 13:42:26 +00:00
/*
* Annotate a token stream with list items with appropriate list tokens
*
* @static
* @method
* @param {[tokens]} Token stream with li tokens
* @returns {[tokens]} Token stream, possibly with additional list tokens
* */
var annotateList = function ( tokens ) {
var out = [], // List of tokens
bstack = [], // Bullet stack, previous element's listStyle
bnext = [], // Next element's listStyle
endtags = []; // Stack of end tags
var commonPrefixLength = function (x, y) {
var minLength = Math.min(x.length, y.length);
for(var i = 0; i < minLength; i++) {
if (x[i] != y[i])
break;
}
return i;
};
var pushList = function ( listName, itemName ) {
out.push({type: 'TAG', name: listName});
out.push({type: 'TAG', name: itemName});
endtags.push({type: 'ENDTAG', name: listName});
endtags.push({type: 'ENDTAG', name: itemName});
};
var popTags = function ( n ) {
for(;n > 0; n--) {
// push list item..
out.push(endtags.pop());
// and the list end tag
out.push(endtags.pop());
}
};
var isDlDd = function (a, b) {
var ab = [a,b].sort();
return (ab[0] === ':' && ab[1] === ';');
};
var doListItem = function ( bs, bn ) {
var prefixLen = commonPrefixLength (bs, bn);
var changeLen = Math.max(bs.length, bn.length) - prefixLen;
2011-11-30 13:42:26 +00:00
var prefix = bn.slice(0, prefixLen);
// emit close tag tokens for closed lists
if (changeLen === 0) {
var itemToken = endtags.pop();
out.push(itemToken);
out.push({type: 'TAG', name: itemToken.name});
endtags.push({type: 'ENDTAG', name: itemToken.name});
} else if ( bs.length == bn.length
&& changeLen == 1
&& isDlDd( bs[prefixLen], bn[prefixLen] ) ) {
// handle dd/dt transitions
out.push(endtags.pop());
if( bn[prefixLen] == ';') {
var newName = 'dt';
} else {
var newName = 'dd';
}
out.push({type: 'TAG', name: newName});
endtags.push({type: 'ENDTAG', name: newName});
} else {
popTags(bs.length - prefixLen);
if (prefixLen > 0 && bn.length == prefixLen ) {
var itemToken = endtags.pop();
out.push(itemToken);
out.push({type: 'TAG', name: itemToken.name});
endtags.push({type: 'ENDTAG', name: itemToken.name});
}
2011-11-30 13:42:26 +00:00
for(var i = prefixLen; i < bn.length; i++) {
switch (bn[i]) {
case '*':
pushList('ul', 'li');
break;
case '#':
pushList('ol', 'li');
break;
case ';':
pushList('dl', 'dt');
break;
case ':':
pushList('dl', 'dd');
break;
default:
throw("Unknown node prefix " + prefix[i]);
}
}
}
};
for (var i = 0, length = tokens.length; i < length; i++) {
var token = tokens[i];
switch ( token.type ) {
case 'TAG':
switch (token.name) {
case 'list':
// ignore token
break;
case 'listItem':
// convert listItem to list and list item tokens
bnext = token.bullets;
doListItem( bstack, bnext );
bstack = bnext;
break;
default:
// pass through all remaining start tags
out.push(token);
break;
}
break;
case 'ENDTAG':
if ( token.name == 'list' ) {
// pop all open list item tokens
popTags(bstack.length);
bstack = [];
2011-11-30 13:42:26 +00:00
} else {
out.push(token);
}
break;
default:
out.push(token);
break;
}
}
return out;
};
/* End static utilities */
/*
* Flags for specific parse environments (inside tables, links etc). Flags
* trigger syntactic stops in the inline_breaks production, which
* terminates inline and attribute matches. Flags merely reduce the number
* of productions needed: The grammar is still context-free as the
* productions can just be unrolled for all combinations of environments
* at the cost of a much larger grammar.
*/
var syntaxFlags = {};
var setFlag = function(flag) {
if (syntaxFlags[flag] !== undefined) {
syntaxFlags[flag]++;
} else {
syntaxFlags[flag] = 1;
2011-11-02 21:09:19 +00:00
}
return true;
};
var clearFlag = function(flag) {
syntaxFlags[flag]--;
return false;
};
// Start position of top-level block
// Could also provide positions for lower-level blocks using a stack.
var blockStart = 0;
// Start position of generic tag production
var tagStartPos = 0;
// cache the input length
var inputLength = input.length;
// pseudo-production that matches at end of input
var isEOF = function (pos) {
return pos === inputLength;
};
// text start position
var textStart = 0;
// hack to support numbered external links ([http://example.com]).
// XXX: Move to token stream transform after templates are expanded!
var linkCount = 1;
// Define block-level tags in JS, so we can use toLowerCase to match tags
// case-independently. This would be quite ugly (and possibly slower) if
// done manually in the grammar.
var block_names = (function () {
var names = [ "p", "table", "td", "tr", "ul", "ol"
, "li", "dl", "dt", "dd", "div", "center"
, "blockquote" ];
var bnames = {};
for(var i = 0, l = names.length; i < l; i++) {
bnames[names[i]] = true;
}
return bnames;
})();
var self = this;
2011-11-02 21:09:19 +00:00
}
start
= e:toplevelblock* newline* {
// end is passed inline as a token, as well as a separate event for now.
// this does not work yet.
//console.log('about to emit' + pp(self));
//self._tokenizer.emit('chunk', [ { type: 'END' } ] );
//self._tokenizer.emit('end');
// Append the end (for obvious reasons this should not
// be part of a stream, only when tokenizing complete
// texts)
return flatten(e);
}
2011-11-02 21:09:19 +00:00
/* All chars that cannot start syntactic structures in the middle of a line
* XXX: ] and other end delimiters should probably only be activated inside
* structures to avoid unnecessarily leaving the text production on plain
* content. */
text_char = [^'<~[{\n\r:\]}|!=]
text = t:text_char+ { return t.join(''); }
/* Explanation of chars
* ' quotes (italic/bold)
* < start of xmlish_tag
* ~ signatures/dates
* [ start of links
* { start of parser functions, transclusion and template args
* \n all sort of block-level markup at start of line
* \r ditto
* h http(s) urls
* n nntp(s) urls
* m mailto urls
*
* ! and | table cell delimiters, might be better to specialize those
* = headings - also specialize those!
*
* The following chars are also included for now, but only apply in some
* contexts and should probably be enabled only in those:
* : separate definition in ; term : definition
* ] end of link
* } end of parser func/transclusion/template arg
*/
2011-11-02 21:09:19 +00:00
urltext = ( t:[^'<~[{\n\rfghimnstw|!:\]} &=]+ { return t.join(''); }
/ htmlentity
/ urllink
// Convert trailing space into &nbsp;
// XXX: This should be moved to a serializer
/ ' ' & ':' { return "\u00a0"; }
/ t:text_char )+
/*
'//', // for protocol-relative URLs, but not in text!
'ftp://',
'git://',
'gopher://',
'http://',
'https://',
'irc://',
'ircs://', // @bug 28503
'mailto:',
'mms://',
'news:',
'nntp://', // @bug 3808 RFC 1738
'svn://',
'telnet://', // Well if we're going to support the above.. -ævar
'worldwind://',
*/
// Old version
//text = t:[A-Za-z0-9,._ "?!\t-]+ { return t.join('') }
// Experimental tweaked version: avoid expensive single-char substrings
// This did not bring the expected performance boost, however.
//text = [A-Za-z0-9,._ -] {
// textStart = pos;
//
// var res = input.substr(textStart - 1, inputLength)
// .match(/[A-Za-z0-9,._ -]+/)[0];
// pos = pos + (res.length - 1);
// return res
// }
2011-11-02 21:09:19 +00:00
htmlentity = "&" c:[#0-9a-zA-Z]+ ";" {
return unentity("&" + c.join('') + ";")
}
2011-11-02 21:09:19 +00:00
space
= s:[ \t]+ { return s.join(''); }
optionalSpaceToken
= s:space* {
if ( s.length ) {
return [{type: 'TEXT', value: s.join('')}];
} else {
return [];
}
}
// Start of line
sol = nl:(newlineToken / & { return pos === 0; } { return [] })
// Eat multi-line comments, so that syntax after still matches as if it
// was actually preceded by a newline
cn:( c:comment n:newline? {
if ( n !== '' ) {
return [c, {type: 'TEXT', value: n}];
} else {
return [c];
}
}
)*
// Eat includeonly/noinclude at start of line, so that start-of-line
// syntax after it still matches
ni:(space* "<" c:"/"? t:("includeonly" / "noinclude") ">" {return [c, t]} )?
{
var niToken = [];
if ( ni !== '') {
if ( ni[0] === '/' ) {
niToken = [{type: 'ENDTAG', name: ni[1]}];
} else {
niToken = [{type: 'TAG', name: ni[1]}];
}
}
return nl.concat(cn, niToken);
}
eof = & { return isEOF(pos); } { return true; }
2011-11-02 21:09:19 +00:00
newline
= '\n' / '\r\n'
2011-11-02 21:09:19 +00:00
newlineToken = newline { return [{type: 'NEWLINE'}] }
eolf = newline / eof
toplevelblock
= & { blockStart = pos; return true; } b:block {
b = flatten(b);
var bs = b[0];
//dp('toplevelblock:' + pp(b));
if (bs.attribs === undefined) {
bs.attribs = [];
}
bs.attribs.push(['data-sourcePos', blockStart + ':' + pos]);
2011-11-30 13:42:26 +00:00
// XXX: only run this for lines that actually need it!
//b.push({type: 'NEWLINE'});
// Move this to a token stream transform!
//console.log('about to emit' + pp(self));
//console.log( pp( result._tokenizer ));
//console.log('emitted chunk' + pp(b));
//return [];
return b;
}
2011-11-02 21:09:19 +00:00
block
= block_lines
/ pre
/ comment &eolf
/ nowiki
/ pre
/ bt:block_tag { return [bt] } // avoid a paragraph if we know that the line starts with a block tag
/ para
/ inlineline // includes generic tags; wrapped into paragraphs in DOM postprocessor
/ s:sol /*{
if (s) {
return [s, {type: 'NEWLINE'}];
} else {
return [{type: 'NEWLINE'}];
}
}*/
block_lines
= s:sol
// eat an empty line before the block
s2:(os:optionalSpaceToken so:sol { return os.concat(so) })?
bl:block_line {
var s2_ = (s2 !== '') ? s2 : [];
return s.concat(s2_, bl);
}
// Block structures with start-of-line wiki syntax
block_line
= h
/// table
/ & [{}|] tl:table_lines { return tl; }
2011-11-02 21:09:19 +00:00
/ lists
// tag-only lines should not trigger pre
/ st:optionalSpaceToken
bt:(bts:block_tag stl:optionalSpaceToken { return bts.concat(stl) })+
&eolf {
return st.concat(bt);
}
/ pre_indent
/ pre
2011-11-02 21:09:19 +00:00
para
= s1:sol s2:sol c:inlineline {
return s1.concat(s2, [{type: 'TAG', name: 'p'}], c);
}
br = space* &newline { return {type: 'SELFCLOSINGTAG', name: 'br'} }
// Syntax stops to limit inline expansion defending on syntactic context
inline_breaks
=
& { // Important hack: disable caching for this production, as the default
// cache key does not take into account flag states!
cacheKey = '';
return true;
}
& { return syntaxFlags['table']; }
( a:(newline [!|] / '||' / '!!' / '|}') { dp("table break" + pp(a) + pos); return true; }
/ & { return syntaxFlags['tableCellArg'] }
"|" { return true }
)
/ & { return (syntaxFlags['colon'] &&
2011-12-07 15:54:55 +00:00
! syntaxFlags.extlink && // example: ; [[Link:Term]] : Definition
! syntaxFlags.linkdesc); } ":" { return true; }
/ & { return syntaxFlags['extlink']; } "]" { return true; }
/ & { return syntaxFlags['linkdesc']; } link_end { return true; }
/ & { return syntaxFlags['h']; } '='+ space* newline { return true; }
/ & { return syntaxFlags['template']; } ('|' / '}}') { return true; }
inline
= c:(urltext / (! inline_breaks (inline_element / . )))+ {
var out = [];
var text = [];
c = flatten(c);
for (var i = 0, l = c.length; i < l; i++) {
var ci = c[i];
if (typeof ci == 'string') {
if(ci !== '') {
text.push(ci);
}
} else {
if (text.length) {
out.push({ type: "TEXT", value: text.join('') });
text = [];
}
out.push(ci);
}
}
if (text.length) {
out.push({ type: 'TEXT', value: text.join('') });
}
//dp('inline out:' + pp(out));
return out;
}
inlineline
= c:(urltext / !inline_breaks (inline_element / [^\n]))+ {
var out = [];
var text = [];
c = flatten(c);
for (var i = 0; i < c.length; i++) {
var ci = c[i]
if (typeof ci == 'string') {
if(ci !== '') {
text.push(ci);
}
} else {
if (text.length) {
out.push({type: 'TEXT', value: text.join('')});
text = [];
}
out.push(ci);
}
}
if (text.length) {
out.push({type: 'TEXT', value: text.join('')});
}
//dp('inlineline out:' + pp(out));
return out;
}
inline_element
= //& { dp('inline_element enter' + input.substr(pos, 10)); return true; }
& '<' ( comment / xmlish_tag )
/ & '{' ( template / tplarg )
/ & '[' ( wikilink / extlink )
/ & "'" quote
/* Headings */
h = & "=" // guard, to make sure '='+ will match.
// XXX: Also check to end to avoid inline parsing?
r:(
s:'='+ // moved in here to make s accessible to inner action
& { return setFlag('h'); }
c:inlineline
e:'='+
spc:(sp:space+ { return {type: 'TEXT', value: sp.join('') } } / comment)*
&eolf
{
clearFlag('h');
var level = Math.min(s.length, e.length);
// convert surplus equals into text
if(s.length > level) {
var extras = s.substr(0, s.length - level);
if(c[0].type == 'TEXT') {
c[0].value = extras + c[0].value;
} else {
c.unshift({type: 'TEXT', value: extras});
}
}
if(e.length > level) {
var extras = e.substr(0, e.length - level),
lastElem = c[c.length - 1];
if(lastElem.type == 'TEXT') {
lastElem.value = lastElem.value + extras;
} else {
c.push({type: 'TEXT', value: extras});
}
}
return [{type: 'TAG', name: 'h' + level}]
.concat(c, [{type: 'ENDTAG', name: 'h' + level}, spc]);
}
/ & { dp('nomatch exit h'); clearFlag('h'); return false } { return null }
) { return r }
2011-11-02 21:09:19 +00:00
pre_indent
= l:pre_indent_line ls:(sol pre_indent_line)* {
return [{type: 'TAG', name: 'pre'}]
.concat( [l], ls
, [{type: 'ENDTAG', name: 'pre'}]);
}
pre_indent_line = space l:inlineline {
return [{type: 'TEXT', value: '\n'}].concat(l);
}
2011-11-02 21:09:19 +00:00
comment
= '<!--' c:comment_chars* ('-->' / eof)
cs:(space* newline space* cn:comment { return cn })* {
return [{ type: 'COMMENT', value: c.join('') }].concat(cs);
}
2011-11-02 21:09:19 +00:00
comment_chars
= c:[^-] { return c; }
/ c:'-' !'->' { return c; }
2011-11-02 21:09:19 +00:00
urllink
= target:url {
return [ { type: 'TAG',
name: 'a',
attribs: [['href', target]] }
, {type: 'TEXT', value: target}
, {type: 'ENDTAG', name: 'a'}
];
}
2011-11-02 21:09:19 +00:00
extlink
= "["
& { return setFlag('extlink'); }
target:url
space*
text:inlineline?
"]" {
clearFlag('extlink');
if ( text == '' ) {
// XXX: Link numbering should be implemented in post-processor.
text = [{type: 'TEXT', value: "[" + linkCount + "]"}];
linkCount++;
}
return [
{
type: 'TAG',
name: 'a',
attribs: [
['href', target],
['data-type', 'external']
],
}
].concat( text
, [{type: 'ENDTAG', name: 'a'}]);
}
/ "[" & { clearFlag('extlink'); return false; }
/* Defaul URL protocols in MediaWiki (see DefaultSettings). Normally these can
* be configured dynamically. */
url_protocol
= '//' // for protocol-relative URLs
/ 'ftp://'
/ 'git://'
/ 'gopher://'
/ 'http://'
/ 'https://'
/ 'irc://'
/ 'ircs://' // @bug 28503
/ 'mailto:'
/ 'mms://'
/ 'news:'
/ 'nntp://' // @bug 3808 RFC 1738
/ 'svn://'
/ 'telnet://' // Well if we're going to support the above.. -ævar
/ 'worldwind://'
// javascript does not support unicode features..
unicode_separator_space = [ \u00A0\u1680\u180E\u2000-\u200A\u202F\u205F\u3000]
2011-11-02 21:09:19 +00:00
urlencoded_char = "%" c0:[0-9a-fA-F] c1:[0-9a-fA-F] {
return decodeURI("%" + c0 + c1)
}
2011-12-08 10:33:23 +00:00
//[^][<>"\\x00-\\x20\\x7F\p{Zs}]
no_punctuation_char = [^ :\]\[\n"'<>\x00-\x20\x7f,.&%\u00A0\u1680\u180E\u2000-\u200A\u202F\u205F\u3000]
2011-11-02 21:09:19 +00:00
url
= proto:url_protocol
2011-12-15 00:26:22 +00:00
rest:( ( !inline_breaks
c:no_punctuation_char
2011-12-15 00:26:22 +00:00
{ return c }
)
/ s:[.:,] !(space / eolf) { return s }
/ htmlentity
/ urlencoded_char
/ [&%] )+
{
return proto + rest.join('');
}
2011-11-02 21:09:19 +00:00
template
= "{{" target:template_target
params:(newline? "|" newline? p:template_param { return p })*
"}}" {
var obj = {
type: 'SELFCLOSINGTAG',
name: 'template',
attribs: [['data-target', target]],
args: {}
};
if (params && params.length) {
var position = 1;
for ( var i = 0, l = params.length; i < l; i++ ) {
var param = params[i];
if ( param[0] === null ) {
obj.args[position] = param[1];
position++;
} else {
2011-12-30 09:35:57 +00:00
// Last value wins for duplicates.
obj.args[param[0]] = param[1];
}
}
// HACK: temporarily also push the args into an attribute
// (just for debugging)
obj.attribs.push(['data-json-args', JSON.stringify(obj.args)]);
}
// Should actually use a self-closing tag here, but the Node HTML5
// parser only recognizes known self-closing tags for now, so use an
// explicit end tag for now.
//console.log(pp(obj));
return obj;
}
2011-11-02 21:09:19 +00:00
// XXX: support template and args in target!
template_target
= h:( !"}}" x:([^|\n]) { return x } )* { return h.join(''); }
2011-11-02 21:09:19 +00:00
template_param
= name:template_param_name space* "=" space* c:template_param_text {
return [name, c];
} / c:template_param_text {
return [null, c];
}
2011-11-02 21:09:19 +00:00
tplarg
= "{{{" name:link_target params:("|" p:template_param { return p })* "}}}" {
var obj = {
type: 'SELFCLOSINGTAG',
name: 'templatearg',
attribs: [['argname', name]]
};
if (params && params.length) {
// HACK, not final.
obj.attribs.push(['data-defaultvalue', params[0][1]]);
obj.attribs.push(['data-json-args', JSON.stringify(params)]);
}
return obj;
}
2011-11-02 21:09:19 +00:00
// FIXME: handle template args and templates in key! (or even parser functions?)
2011-11-02 21:09:19 +00:00
template_param_name
= h:( !"}}" x:([^=|\n]) { return x } )* { return h.join(''); }
2011-11-02 21:09:19 +00:00
template_param_text
= & { return setFlag('template') }
il:inline+ {
clearFlag('template');
return il;
}
/ & { clearFlag('template'); return false; }
2011-11-02 21:09:19 +00:00
// TODO: handle link prefixes as in al[[Razi]]
wikilink
= "[["
! url
2011-12-15 00:12:58 +00:00
target:link_target
2011-12-30 09:35:57 +00:00
lcontent:( "|" lt:link_text { return lt } )*
2011-12-15 00:12:58 +00:00
"]]"
// XXX In real MediaWiki, this is a language-dependent positive character
// class. Can we work out a static negative class instead?
trail:(! [ \t(),.:-] tc:text_char { return tc })* {
var obj = {
type: 'TAG',
name: 'a',
attribs: [
['data-type', 'internal']
]
2011-12-30 09:35:57 +00:00
},
textTokens = [];
2011-11-18 14:48:32 +00:00
obj.attribs.push(['href', target]);
2011-12-30 09:35:57 +00:00
if (lcontent && lcontent.length) {
textTokens = lcontent;
if (trail) {
textTokens.push( { type: 'TEXT', value: trail.join('') } );
2011-12-30 09:35:57 +00:00
}
2011-11-18 14:48:32 +00:00
} else {
if (trail) {
target += trail.join('');
}
2011-12-30 09:35:57 +00:00
textTokens = [{type: 'TEXT', value: target}];
}
return [obj].concat(textTokens, [{type: 'ENDTAG', name: 'a'}]);
}
2011-11-02 21:09:19 +00:00
link_target
= h:( c:[^|%\n\]]+ { return c.join('') } // quickly eat anything unsuspicious
/ !"]]"
hi:(
[^|%\n]
/ urlencoded_char
/ '%'
) { return hi }
)* { return h.join(''); }
2011-11-02 21:09:19 +00:00
link_text
= & { return setFlag('linkdesc'); }
h:inlineline
{
clearFlag('linkdesc');
return h;
}
/ & { clearFlag('linkdesc'); return false }
link_end = "]]"
2011-11-02 21:09:19 +00:00
/* Generic quote production for italic and bold, further processed in a token
* stream transformation in doQuotes. Relies on NEWLINE tokens being emitted
* for each line of text to balance quotes per line.
*
* We are not using a simple pair rule here as we need to support mis-nested
* bolds/italics and MediaWiki's special heuristics for apostrophes, which are
* all not context free. */
quote = "''" x:"'"* {
return {
type: 'TAG',
name : 'mw-quote', // Will be consumed in token transforms
value: "''" + x.join('')
}
}
2011-11-02 21:09:19 +00:00
/* XXX: Extension tags can require a change in the tokenizer mode, which
* returns any text between extension tags verbatim. For now, we simply
* continue to parse the contained text and return the tokens. The original
* input source can be recovered from the source positions added on tag
* tokens. This won't however work in all cases. For example, a comment start
* (<!--) between extension tags would cause the remaining text to be consumed
* as a comment. To avoid this, we might need to look ahead for the end tag
* and limit the content parsing to this section. */
xmlish_tag = nowiki / generic_tag
pre
= "<pre"
attribs:generic_attribute*
">"
ts:(t1:[^<]+ { return {type:'TEXT',value:t1.join('')} }
/ nowiki
/ !"</pre>" t2:. {return {type:'TEXT',value:t2}})+
("</pre>" / eof) {
// return nowiki tags as well?
//console.log('inpre');
return [ {type: 'TAG', name: 'pre', attribs: attribs} ]
.concat(ts, [{type: 'ENDTAG', name: 'pre'}]);
}
/ "</pre>" { return {type: 'TEXT', value: "</pre>"}; }
nowiki
= "<nowiki>" nc:nowiki_content "</nowiki>" {
// console.log(pp(nc));
return nc;
}
/ "<nowiki>" {
//console.log('nowiki fallback');
return [{type: 'TEXT', value: '<nowiki>'}];
}
/ "</nowiki>" { return [{type: 'TEXT', value: '</nowiki>'}]; }
nowiki_content
= ts:( t:[^<]+ { return t.join('') }
/ "<pre" p0:space* p1:[^>]* ">" p2:nowiki_content "</pre>" {
//console.log('nested pre in nowiki');
return ["<pre"].concat(p0, p1, [">"], [p2[0].value], ["</pre>"]).join('');
}
/ (!("</nowiki>" / "</pre>") c:. {return c})
)* {
// return nowiki tags as well?
return [{type: 'TEXT', value: ts.join('')}];
}
// See http://dev.w3.org/html5/spec/Overview.html#syntax-tag-name and
// following paragraphs
block_tag
= "<" end:"/"? name:(cs:[a-zA-Z]+ { return cs.join('') })
attribs:generic_attribute*
selfclose:"/"?
">" {
if (block_names[name.toLowerCase()] !== true) {
// abort match if tag is not block-level
return null;
}
var res = {name: name, attribs: attribs};
if ( end != '' ) {
res.type = 'ENDTAG';
} else if ( selfclose != '' ) {
res.type = 'SELFCLOSINGTAG';
} else {
res.type = 'TAG';
}
return [res];
}
/* Generic XML-like tags
*
* These also cover extensions (including Cite), which will hook into the
* token stream for further processing. The content of extension tags is
* parsed as regular inline, but the source positions of the tag are added
* to allow reconstructing the unparsed text from the input. */
// See http://dev.w3.org/html5/spec/Overview.html#syntax-tag-name and
// following paragraphs
generic_tag
= "<"
& { tagStartPos = pos; return true; } // remember the start position of this tag
end:"/"? name:[0-9a-zA-Z]+
attribs:generic_attribute*
space*
selfclose:"/"?
">" {
var res = {name: name.join(''), attribs: attribs};
if ( end != '' ) {
res.type = 'ENDTAG';
} else if ( selfclose != '' ) {
res.type = 'SELFCLOSINGTAG';
} else {
res.type = 'TAG';
}
res.attribs.push(['data-sourceTagPos', (tagStartPos - 1) + ":" + pos]);
return res;
}
generic_attribute
= s:space*
name:generic_attribute_name
value:(space*
v:generic_attribute_value { return v })?
{
if ( value !== '' ) {
return [name, value];
} else {
return [name,''];
}
}
// http://dev.w3.org/html5/spec/Overview.html#attributes-0, and we also
// disallow newlines, | and {.
generic_attribute_name
= n:[^ \t\0/"'>=\n|{]+ {
return n.join('');
}
generic_attribute_value
= "=" space* v:att_value {
return v;
}
// XXX: attributes can contain templates and template args!!
att_value
= t:(!inline_breaks c:[^ \t'"<>='\n] { return c } )+ {
return t.join('');
}
// XXX: is "\"" also valid html? or just Wikitext?
/ "'" t:[^'>]* "'" { return unquote("'", t.join('')); }
/ '"' t:[^">]* '"' { return unquote('"', t.join('')); }
2011-11-02 21:09:19 +00:00
/* Lists */
lists = e:(dtdd / li) es:(sol (dtdd / li))*
{
return annotateList( [ { type: 'TAG', name: 'list'} ]
2011-11-30 13:42:26 +00:00
.concat(flatten([e].concat(es))
,[{ type: 'ENDTAG', name: 'list' }]));
2011-11-02 21:09:19 +00:00
}
li = bullets:list_char+
c:inlineline?
&eolf
2011-11-02 21:09:19 +00:00
{
if ( c == '' )
c = [];
return [ { type: 'TAG',
name: 'listItem',
bullets: bullets }
, c ];
2011-11-02 21:09:19 +00:00
}
dtdd
= bullets:(!(";" !list_char) list_char)*
";"
& {return setFlag('colon');}
c:inlineline
":"
// Fortunately dtdds cannot be nested, so we can simply set the flag
// back to 0 to disable it.
& {syntaxFlags['colon'] = 0; return true;}
d:inlineline
&eolf {
// Convert trailing space into &nbsp;
// XXX: This should be moved to a serializer
//var clen = c.length;
//if (clen && c[clen - 1].type === 'TEXT') {
// var val = c[clen - 1].value;
// if(val.length && val[val.length - 1] == ' ') {
// c[clen - 1].value = val.substr(0, val.length - 1) + "\u00a0";
// }
//}
return [ { type: 'TAG', name: 'listItem', bullets: bullets + ";" } ]
.concat( c
,[{ type: 'TAG', name: 'listItem', bullets: bullets + ":" } ]
, d );
2011-11-02 21:09:19 +00:00
}
// Fall-back case to clear the colon flag
/ & { return true; } { syntaxFlags['colon'] = 0; return null; }
2011-11-02 21:09:19 +00:00
list_char = [*#:;]
2011-12-22 12:37:24 +00:00
/**
* Tables
*
* Table productions are geared to support independent parsing of fragments in
* templates (the common table start / row / table end use case). The tokens
* produced by these fragments then match up to a table while building the
* DOM tree. For similar reasons, table rows do not emit explicit end tag
* tokens.
*
* The separate table_lines production is faster than moving those productions
* directly to block_lines.
* */
table_lines
= & { return setFlag('table'); }
tl:table_line
tls:( s:sol tl2:table_line { return s.concat(tl2); } )* {
clearFlag('table');
//console.log('table_lines: ' + pp(tl.concat(tls)));
return tl.concat( tls );
}
/ & { return clearFlag('table'); }
2011-12-22 12:37:24 +00:00
// This production assumes start-of-line position!
table_line
= table_start_tag
/ table_row_tag
/ table_data_tags
/ table_heading_tags
/ table_caption_tag
/ table_end_tag
table_start_tag
= "{|"
ta:generic_attribute*
space*
te:table_end_tag? // can occur somewhere in the middle of the line too
{
var tok = [{type: 'TAG', name: 'table'}];
if ( ta )
tok[0].attribs = ta;
if ( te )
tok = tok.concat(te);
return tok;
}
table_caption_tag
= "|+"
c:inline* {
return [{type: 'TAG', name: 'caption'}]
.concat( c, [{type: 'ENDTAG', name: 'caption'}]);
}
table_row_tag
= //& { console.log("table row enter"); return true; }
"|-"
a:generic_attribute*
space*
2011-12-22 11:58:32 +00:00
// handle tables with missing table cells after a row
td:( s:sol ![|!] tdt:table_data_tag { return s.concat(tdt); } )?
{
// We rely on our tree builder to close the row as needed. This is
// needed to support building tables from fragment templates with
// individual cells or rows.
2011-12-22 11:58:32 +00:00
var trToken = [{type: 'TAG', name: 'tr', attribs: a}];
if ( !td ) {
return trToken;
} else {
return trToken.concat(td);
}
}
table_data_tags
2011-12-22 11:58:32 +00:00
= "|"
td:table_data_tag
tds:( "||" tdt:table_data_tag { return tdt } )* {
return td.concat(tds);
}
2011-11-02 21:09:19 +00:00
table_data_tag
= //& { dp("table_data enter, pos=" + pos + input.substr(pos,10)); return true; }
! [}+-]
a:table_cell_args?
//& { console.log("past attrib, pos=" + pos + input.substr(pos,10)); return true; }
// use inline_breaks to break on tr etc
td:( !inline_breaks
//& { dp("table_data 2, pos=" + pos + input.substr(pos,10)); return true; }
b:block { return b } )*
{
if ( a == '' ) {
a = [];
}
//dp("table data result: " + pp(td) + ", attribts: " + pp(a));
return [{ type: 'TAG', name: 'td', attribs: a}]
.concat( td, [{type: 'ENDTAG', name: 'td'}] );
}
table_heading_tags
= "!"
th:table_heading_tag
ths:( "!!" tht:table_heading_tag { return tht } )* {
return th.concat(ths);
}
table_heading_tag
= a:table_cell_args?
c:inline {
if ( a == '' ) {
a = [];
}
return [{type: 'TAG', name: 'th', attribs: a}]
.concat( c, [{type: 'ENDTAG', name: 'th'}] );
}
table_end_tag
= "|}" {
var tok = [{type: 'ENDTAG', name: 'table'}];
return tok;
}
table_cell_args
= & { return setFlag('tableCellArg'); }
as:generic_attribute+ space* "|" !"|" {
clearFlag('tableCellArg');
return as;
}
/ & { return clearFlag('tableCellArg'); }
2011-12-22 12:37:24 +00:00
/* Tables, full-table version (no support for table fragments from templates)
* Only left here for comparison purposes with the above productions. Remove
* when those work as intended! */
2011-11-02 21:09:19 +00:00
table
= tas:table_start space* c:table_caption? b:table_body? te:table_end {
var res = {type: 'TAG', name: 'table'}
2011-11-02 21:09:19 +00:00
var body = b !== '' ? b : [];
dp("body: " + pp(body));
2011-11-02 21:09:19 +00:00
if (tas.length > 0) {
// FIXME: actually parse and build structure
//res.attribs = [['data-unparsed', tas.join('')]];
res.attribs = tas;
2011-11-02 21:09:19 +00:00
}
if (c != '') {
var caption = [{type: 'TAG', name: 'caption'}]
.concat(c, [{type: 'ENDTAG', name: 'caption'}], te);
} else {
var caption = [];
}
//dp(pp(res));
return [res].concat(caption, body,
[{type: 'ENDTAG', name: 'table'}]);
2011-11-02 21:09:19 +00:00
}
table_start
= "{|"
res:(
& { setFlag('table'); return true; }
ta:generic_attribute*
{
dp("table_start " + pp(ta) + ", pos:" + pos);
return ta;
}
/ & { clearFlag('table'); return false; } { return null; }
) { return res }
2011-11-02 21:09:19 +00:00
table_caption
= n:newlineToken
"|+" c:inline* {
return n.concat(c);
}
2011-11-02 21:09:19 +00:00
table_body
= //& { dp("table_body enter"); return true; }
2011-11-02 21:09:19 +00:00
firstrow:table_firstrow otherrows:table_row* {
/* dp('table first and otherrows: '
* + pp([firstrow].concat(otherrows))); */
2011-11-02 21:09:19 +00:00
return [firstrow].concat(otherrows);
}
/ otherrows:table_row* {
//dp('table otherrows: ' + pp(otherrows));
2011-11-02 21:09:19 +00:00
return otherrows;
}
table_firstrow
= td:(table_data / table_heading)+ {
//dp('firstrow: ' + pp(td));
return [{ type: 'TAG', name: 'tr' }]
.concat(td, [{type: 'ENDTAG', name: 'tr'}]);
2011-11-02 21:09:19 +00:00
}
table_row
= //& { dp("table row enter"); return true; }
n:newlineToken
"|-"
a:(as:generic_attribute+ space* "|" !"|" { return as } )?
space*
td:(table_data / table_heading)* {
return n.concat([{type: 'TAG', name: 'tr'}]
, td, [{type: 'ENDTAG', name: 'tr'}]);
}
2011-11-02 21:09:19 +00:00
table_data
= //& { dp("table_data enter, pos=" + pos + input.substr(pos,10)); return true; }
n:("||" { return [] } / nt:newlineToken ("|" / !'!') { return nt })
! [}+-]
//& { dp('before attrib, pos=' + pos); return true; }
a:(as:generic_attribute+ space* "|" !"|" { return as } )?
//& { dp('past attrib, pos=' + pos); return true; }
// use inline_breaks to break on tr etc
td:(!inline_breaks
//& { dp("table_data 2, pos=" + pos + input.substr(pos,10)); return true; }
b:block { return b })* {
if ( a == '' ) {
a = [];
}
//dp("table data result: " + pp(td) + ", attribts: " + pp(a));
return n.concat( [{ type: 'TAG', name: 'td', attribs: a}]
, td, [{type: 'ENDTAG', name: 'td'}] );
}
2011-11-02 21:09:19 +00:00
table_heading
= n:("!!" { return [] } / nl:newlineToken "!" { return nl })
2011-12-16 19:22:13 +00:00
a:(as:generic_attribute+ "|" !"|" { return as } )?
c:inline {
if ( a == '' ) {
a = [];
}
return n.concat( [{type: 'TAG', name: 'th', attribs: a}]
, c, [{type: 'ENDTAG', name: 'th'}]);
}
thtd_attribs
// In particular, do not match [|\n]
= a:(text / ! inline_breaks c:[="':;/,. -] { return c } )+ "|" ! "|" {
return a;
}
2011-11-02 21:09:19 +00:00
table_end
= nt:newlineToken? ( "|}" / eof ) {
clearFlag('table');
if(nt)
return nt;
else
return [];
}
2011-11-02 21:09:19 +00:00
/* Tabs do not mix well with the hybrid production syntax */
/* vim: set filetype=javascript expandtab ts=4 sw=4 cindent : */