pegTokenizer.pegjs.txt
3df9b319977fmaster

pegTokenizer.pegjs.txt
View Options

			/**
			* Combined Wiki (MediaWiki) and HTML tokenizer based on pegjs. Emits several
			* chunks of tokens (one chunk per top-level block matched) and eventually an
			* end event. Tokens map to HTML tags as far as possible, with custom tokens
			* used where further processing on the token stream is needed.
			*/
			{

			var pegIncludes = options.pegIncludes;
			var DU = pegIncludes.DOMUtils;
			var Util = pegIncludes.Util;
			var PegTokenizer = pegIncludes.PegTokenizer;
			var defines = pegIncludes.defines;
			var constants = pegIncludes.constants;
			var tu = pegIncludes.tu;

			// define some constructor shortcuts
			var KV = defines.KV;
			var TagTk = defines.TagTk;
			var SelfclosingTagTk = defines.SelfclosingTagTk;
			var EndTagTk = defines.EndTagTk;
			var NlTk = defines.NlTk;
			var CommentTk = defines.CommentTk;
			var EOFTk = defines.EOFTk;

			var inlineBreaks = tu.inlineBreaks;
			var stops = new tu.SyntaxStops();

			/*
			* Emit a chunk of tokens to our consumers. Once this has been done, the
			* current expression can return an empty list (true).
			*/
			var emitChunk = function(tokens) {
			// Shift tsr of all tokens by the pipeline offset
			Util.shiftTokenTSR( tokens, options.pipelineOffset );
			options.env.log("trace/peg", options.pegTokenizer.pipelineId, "----> ", tokens);
			// limit the size of individual chunks
			var chunkLimit = 100000;
			if (tokens.length > chunkLimit) {
			var i = 0;
			var l = tokens.length;
			while (i < l) {
			options.cb(tokens.slice(i, i + chunkLimit));
			i += chunkLimit;
			}
			} else {
			options.cb(tokens);
			}
			};

			}

			/*********************************************************
			* The top-level rule
			*********************************************************/

			start
			= tlb* newline* {
			// end is passed inline as a token, as well as a separate event for now.
			emitChunk( [ new EOFTk( ) ] );
			return true;
			}

			/*
			* Redirects can only occur as the first thing in a document. See
			* WikitextContent::getRedirectTarget()
			*/
			redirect
			= rw:redirect_word
			sp:$space_or_newline*
			c:$( ":" space_or_newline* )?
			wl:wikilink
			{
			var link = wl[0];
			if (!link \|\| link.constructor === String) {
			peg$currPos = peg$reportedPos;
			return peg$FAILED;
			}
			// Set the wikilink's tsr to be zero length. That token is synthetic and
			// the full tsr range is used by the redirect token. Anything with a zero
			// width will never be serialized using selser, which is what we want here
			// as there is no source for it.
			link.dataAttribs.tsr[0] = link.dataAttribs.tsr[1];

			if (sp) { rw += sp; }
			if (c) { rw += c; }
			// Build a redirect token
			var redirect = new SelfclosingTagTk('mw:redirect',
			[Util.lookupKV(link.attribs, 'href')],
			{
			src: rw,
			tsr: [peg$reportedPos, peg$currPos],
			linkTk: link
			});
			return redirect;
			}

			// This rule is exposed as a start rule.
			generic_attributes = generic_attribute*

			/* The 'redirect' magic word.
			* The leading whitespace allowed is due to the PHP trim() function.
			*/
			redirect_word = sp:$[ \t\n\r\0\x0b]* rw:$(!space_or_newline ![:\[] .)+
			{
			if ( options.env.conf.wiki.getMagicWordMatcher( 'redirect' ).test( rw ) ) {
			return sp + rw;
			}
			peg$currPos = peg$reportedPos;
			return peg$FAILED;
			}

			/*
			* This rule exists to support tokenizing the document in chunks.
			* It stops tokenization after each block and yields to the node.js
			* event-loop to schedule other pending event handlers.
			*
			* It needs to keep track of sol-state so when tokenization resumes,
			* it knows whether it is in sol-state or not.
			*/
			toplevelblock =
			tlb save_sol_state {
			var newOffset = peg$currPos;

			// Trick the tokenizer into ending parsing
			peg$currPos = input.length;

			return { eof: false, newOffset: newOffset };
			}
			/ newline* eof {
			// Clear saved sol state!
			options.pegTokenizer.savedSOL = null;
			emitChunk( [ new EOFTk( ) ] );

			return { eof: true };
			}

			save_sol_state =
			& (s:sol? { options.pegTokenizer.savedSOL = !!s; return true; })

			/*
			* A document (start rule) is a sequence of toplevelblocks. Tokens are
			* emitted in chunks per toplevelblock to avoid buffering the full document.
			*/
			tlb
			= !eof b:block {
			// Clear the tokenizer's backtracking cache after matching each
			// toplevelblock. There won't be any backtracking as a document is just a
			// sequence of toplevelblocks, so the cache for previous toplevelblocks
			// will never be needed.
			var end = options.startOffset \|\| 0;
			for (var i = options.prevOffset \|\| 0; i < end; i++) {
			peg$cache[i] = undefined;
			}

			var tokens;
			if ( Array.isArray(b) && b.length ) {
			tokens = tu.flattenIfArray(b);
			} else if (b && b.constructor === String) {
			tokens = [b];
			}

			// Emit tokens for this toplevelblock. This feeds a chunk to the parser pipeline.
			if ( tokens ) {
			emitChunk( tokens );
			}

			// We don't return any tokens to the start rule to save memory. We
			// just emitted them already to our consumers.
			return true;
			}

			/*
			* The actual contents of each block.
			*/
			block
			= &sof r:redirect {return [r];} // has to be first alternative; otherwise gets parsed as a <ol>
			/ block_lines
			/ & '<' rs:( pre // tag variant can start anywhere
			/ c:comment &eolf { return c; }
			/ nowiki
			// avoid a paragraph if we know that the line starts with a block tag
			/ bt:block_tag { return [bt]; }
			) { return rs; }
			/ paragraph
			// Inlineline includes generic tags; wrapped into paragraphs in token
			// transform and DOM postprocessor
			/ inlineline
			/ s:sol !inline_breaks { return s; }

			/*
			* A block nested in other constructs. Avoid eating end delimiters for other
			* constructs by checking against inline_breaks first.
			*/
			nested_block = !inline_breaks b:block { return b; }

			nested_block_line = bs:(!sol !inline_breaks b:block { return b; })* {
			return tu.flattenIfArray(bs);
			}

			/*
			* The same, but suitable for use inside a table construct.
			* Doesn't match table_heading_tag, table_row_tag, table_data_tag,
			* table_caption tag, or table_end_tag, although it does allow
			* table_start_tag (for nested tables).
			*/
			nested_block_in_table
			=
			// avoid recursion via nested_block_in_table, as that can lead to stack
			// overflow in large tables
			// See https://bugzilla.wikimedia.org/show_bug.cgi?id=57670
			& { return stops.push('tableDataBlock', true ); }
			// XXX: don't rely on a lame look-ahead like this; use syntax stops
			// instead, so that multi-line th content followed by a line prefixed with
			// a comment is also handled. Alternatively, implement a sol look-behind
			// assertion accepting spaces and comments.
			!(sol (space* sol)? space* (pipe / "!")) b:nested_block {
			stops.pop('tableDataBlock');
			return b;
			}
			/ & { return stops.pop('tableDataBlock'); }

			/*
			* Line-based block constructs.
			*/
			block_lines
			= s:sol
			// eat an empty line before the block
			s2:(os:optionalSpaceToken so:sol { return os.concat(so); })?
			bl:block_line {
			var s2_ = (s2 !== null) ? s2 : [];
			return s.concat(s2_, bl);
			}

			/*
			* Block structures with start-of-line wiki syntax
			*/
			block_line
			= h
			/ list_item
			/ st:space_or_newline*
			r:( & [ <{}\|!] tl:table_lines { return tl; }
			// tag-only lines should not trigger pre either
			/ bts:(bt:block_tag stl:optionalSpaceToken { return bt.concat(stl); })+
			&eolf { return bts; }
			) {
			return st.concat(r);
			}
			/ ! { return stops.counters.nopre; } pi:pre_indent { return pi; }
			/ pre
			/ // Horizontal rules
			"----" d:"-"*
			// Check if a newline or content follows
			lineContent:( &sol { return undefined; } / { return true; } ) {
			if (d.length > 0) {
			return new SelfclosingTagTk( "hr", [],
			{
			tsr: [peg$reportedPos, peg$currPos],
			extra_dashes: d.length,
			lineContent: lineContent
			} );
			} else {
			return new SelfclosingTagTk( "hr", [],
			{
			tsr: [peg$reportedPos, peg$currPos],
			lineContent: lineContent
			} );
			}
			}

			/*
			* A paragraph. We don't emit 'p' tokens to avoid issues with template
			* transclusions, <p> tags in the source and the like. Instead, we perform
			* some paragraph wrapping on the token stream and the DOM.
			*/
			paragraph
			= s1:sol s2:sol c:inlineline {
			return s1.concat(s2, /* [new TagTk('p')],*/ c);
			}

			br = s:optionalSpaceToken &newline {
			return s.concat(
			[
			new SelfclosingTagTk( 'br', [], {tsr: [peg$reportedPos, peg$currPos]} )
			]
			);
			}

			inline_breaks
			= & { return inlineBreaks( input, peg$currPos, stops ); }

			pre_start = "<" pre_tag_name [^>]* ">"

			inline
			= c:(urltext / (!inline_breaks !pre_start r:(inline_element / . ) { return r; }))+ {
			return tu.flattenStringlist( c );
			}

			inlineline
			= c:(urltext
			/ !{ return inlineBreaks( input, peg$currPos, stops ); } // inline_breaks
			!pre_start
			r:(inline_element / [^\r\n]) { return r; })+ {
			return tu.flattenStringlist( c );
			}

			inline_element
			= //& { dp('inline_element enter' + input.substr(peg$currPos, 10)); return true; }
			& '<' r:( nowiki
			/ xmlish_tag
			/ comment
			) { return r; }
			/// & '{' ( & '{{{{{' template / tplarg / template )
			/ & '{' r:tplarg_or_template_or_broken { return r; }
			/ & '}' r:broken_template { return r; }
			/// & '{' ( tplarg / template )
			// Eat three opening brackets as text, but handle '[[[[' differently
			// so, that '[[[[Foo]]]]' parses as '[[<a..>Foo</a>]]'
			/ (!'[' / sol) '[[[' !'[' { return '[[['; }
			/ & '[' r:( wikilink / extlink ) { return r; }
			/ & "'" r:quote { return r; }

			/* Headings */

			h = & "=" // guard, to make sure '='+ will match.
			// XXX: Also check to end to avoid inline parsing?
			r:(
			s:$'='+ // moved in here to make s accessible to inner action
			& { return stops.inc('h'); }
			c:nested_block_line
			e:$'='+
			endTPos:({ return peg$currPos; })
			spc:(spaces / comment)*
			&eolf
			{
			stops.dec('h');
			var level = Math.min( s.length, e.length );
			level = Math.min( 6, level );
			// convert surplus equals into text
			if (s.length > level) {
			var extras1 = s.substr(0, s.length - level);
			if (c[0].constructor === String) {
			c[0] = extras1 + c[0];
			} else {
			c.unshift( extras1 );
			}
			}
			if (e.length > level) {
			var extras2 = e.substr(0, e.length - level);
			var lastElem = c[c.length - 1];
			if (lastElem.constructor === String) {
			c[c.length - 1] += extras2;
			} else {
			c.push( extras2 );
			}
			}

			return [new TagTk( 'h' + level, [], { tsr: [peg$reportedPos, peg$reportedPos + level] } )]
			.concat(c, [
			new EndTagTk( 'h' + level, [],
			{ tsr: [endTPos - level, endTPos]} ),
			spc
			]);
			}
			/ & { stops.dec('h'); return false; }
			) { return r; }


			/* Comments */

			// The php parser does a straight str.replace(/<!--((?!-->).)*-->/g, "")
			// but, as always, things around here are a little more complicated.
			//
			// We accept the same comments, but because we emit them as HTML comments
			// instead of deleting them, we have to encode the data to ensure that
			// we always emit a valid HTML5 comment. See the encodeComment helper
			// for further details.

			comment
			= '<!--' c:$( !"-->" . )* ('-->' / eof) {
			var data = DU.encodeComment( c );
			return [new CommentTk( data, { tsr: [peg$reportedPos, peg$currPos] } )];
			}


			// Behavior switches. See:
			// https://www.mediawiki.org/wiki/Help:Magic_words#Behavior_switches
			behavior_switch
			= bs:$('__' behavior_text '__') {
			return [ new SelfclosingTagTk( 'behavior-switch', [ new KV('word', bs) ], {
			tsr: [ peg$reportedPos, peg$currPos ],
			src: bs
			}) ];
			}

			// Instead of defining a charset, php's doDoubleUnderscore concats a regexp of
			// all the language specific aliases of the behavior switches and then does a
			// match and replace. Just be as permissive as possible and let the
			// BehaviorSwitchPreprocessor back out of any overreach.
			behavior_text = $( !'__' [^'"<~[{\n\r:;\]}\|!=] )+

			/**************************************************************
			* External (bracketed and autolinked) links
			**************************************************************/

			autolink
			= ! { return stops.onStack('extlink'); }
			// this must be a word boundary, so previous character must be non-word
			! { return /\w/.test(input[peg$currPos - 1] \|\| ''); }
			r:(
			// urllink, inlined
			target:autourl {
			var res = [ new SelfclosingTagTk( 'urllink', [new KV('href', target)], { tsr: [peg$reportedPos, peg$currPos] } ) ];
			return res;
			}
			/ autoref
			/ isbn) { return r; }

			extlink
			= ! { return stops.onStack('extlink'); } // extlink cannot be nested
			r:(
			"["
			& { return stops.push('extlink', true); }
			target:extlink_preprocessor_text
			& { return Util.isProtocolValid( target, options.env ); }
			sp:$( space / [\u00A0\u1680\u180E\u2000-\u200A\u202F\u205F\u3000] )*
			targetOff:({ return peg$currPos; })
			content:(
			t1:(
			& { return stops.push('pipe', false); }
			t:inlineline { stops.pop('pipe'); return t; }
			/ ! { return stops.pop('pipe'); }
			) { return t1; }
			)?
			"]" {
			stops.pop('extlink');
			// if ( text === '' ) {
			// // XXX: Link numbering should be implemented in post-processor.
			// text = [ "[" + linkCount + "]" ];
			// linkCount++;
			// }
			return [
			new SelfclosingTagTk( 'extlink', [
			new KV('href', target),
			new KV('mw:content', content),
			new KV('spaces', sp)
			], {
			targetOff: targetOff,
			tsr: [peg$reportedPos, peg$currPos],
			contentOffsets: [targetOff, peg$currPos - 1]
			})
			];
			}
			/ br:"[" & { return stops.pop('extlink'); } { return br; }
			) { return r; }

			autoref
			= ref:('RFC' / 'PMID') space_or_newline+ identifier:$[0-9]+ end_of_word
			{
			var base_urls = {
			'RFC': '//tools.ietf.org/html/rfc%s',
			'PMID': '//www.ncbi.nlm.nih.gov/pubmed/%s?dopt=Abstract',
			};
			var url = tu.sprintf(base_urls[ref], identifier);

			return [
			new SelfclosingTagTk('extlink', [
			new KV('href', tu.sprintf(base_urls[ref], identifier)),
			new KV('mw:content', [ref, identifier].join(' ')),
			new KV('typeof', 'mw:ExtLink/' + ref)
			],
			{ stx: "magiclink", tsr: [peg$reportedPos, peg$currPos] })
			];
			}

			isbn
			= 'ISBN' space_or_newline+
			head:[0-9]
			digits:$( [- ] &[0-9] / [0-9] )+
			tail:$([- ]? [xX])?
			end_of_word
			{
			// TODO: round-trip non-decimals too!
			var isbn = [head, digits, tail].join('');
			var isbncode = isbn.replace(/[^\dX]/g, '');

			// ISBNs can only be 10 or 13 chars long
			if ([10, 13].indexOf(isbncode.length) === -1) {
			// just return the string
			return [ text() ];
			}

			return [
			new SelfclosingTagTk( 'extlink', [
			new KV('href', 'Special:BookSources/' + isbncode),
			new KV('mw:content', 'ISBN ' + isbn),
			new KV('typeof', 'mw:ExtLink/ISBN')
			],
			{stx: "magiclink", tsr: [peg$reportedPos, peg$currPos]})
			];
			}


			/* Default URL protocols in MediaWiki (see DefaultSettings). Normally
			* these can be configured dynamically. */

			url_protocol =
			& { return Util.isProtocolValid(input.substr(peg$currPos), options.env); }
			h:$[a-zA-Z\/]+ c:':'? s:'//'?
			{
			if (c) {
			h += c;
			}
			if (s) {
			h += s;
			}
			return h;
			}

			// javascript does not support unicode features..
			unicode_separator_space = [ \u00A0\u1680\u180E\u2000-\u200A\u202F\u205F\u3000]


			urlencoded_char = "%" c0:[0-9a-fA-F] c1:[0-9a-fA-F] {
			try {
			return decodeURI("%" + c0 + c1);
			} catch ( e ) {
			// Reject the match, and allow other fall-back rules to have a
			// go at it.
			peg$currPos = peg$reportedPos;
			return peg$FAILED;
			}
			}

			//[^][<>"\\x00-\\x20\\x7F\p{Zs}]

			// no punctuation, and '{<' to trigger directives
			no_punctuation_char = [^ :\]\[\r\n"'<>\x00-\x20\x7f,.&%\u00A0\u1680\u180E\u2000-\u200A\u202F\u205F\u3000{]

			// this is the general url rule
			// on the PHP side, the path part matches EXT_LINK_URL_CLASS
			// which is '[^][<>"\\x00-\\x20\\x7F\p{Zs}]'
			// the 's' and 'r' pieces below match the characters in
			// EXT_LINK_URL_CLASS which aren't included in no_punctuation_char
			url
			= proto:url_protocol
			addr:( ipv6_address / ipv4_address )?
			path:( ( !{ return inlineBreaks( input, peg$currPos, stops ); } // inline_breaks
			c:no_punctuation_char
			{ return c; }
			)
			/ s:[.:,'] { return s; }
			/ comment
			/ tplarg_or_template
			/ ! ( "&" ( [lL][tT] / [gG][tT] ) ";" )
			r:(
			& "&" he:htmlentity { return he; }
			/ [&%{]
			) { return r; }
			)+
			{
			proto += addr \|\| '';
			return tu.flattenString( [proto].concat( path ) );
			}

			// this is the somewhat-restricted rule used in autolinks
			// See Parser::doMagicLinks and Parser.php::makeFreeExternalLink.
			// The `path` portion matches EXT_LINK_URL_CLASS, as in the general
			// url rule. As in PHP, we do some fancy fixup to yank out
			// trailing punctuation, perhaps including parentheses.
			// The 's' and 'r' pieces match the characters in EXT_LINK_URL_CLASS
			// which aren't included in no_punctuation_char
			autourl
			= &{ return stops.push( 'autourl', { sawLParen: false } ); }
			proto:url_protocol
			addr:( ipv6_address / ipv4_address )?
			path:( ( !{ return inlineBreaks( input, peg$currPos, stops ); } // inline_breaks
			! "("
			c:no_punctuation_char
			{ return c; }
			)
			/ "(" { stops.onStack( 'autourl' ).sawLParen = true; return "("; }
			/ s:[.:,'] { return s; }
			/ comment
			/ tplarg_or_template
			/ ! ( "&" ( [lL][tT] / [gG][tT] ) ";" )
			r:(
			& "&" he:htmlentity { return he; }
			/ [&%{]
			) { return r; }
			)+
			{
			// as in Parser.php::makeFreeExternalLink, we're going to
			// yank trailing punctuation out of this match.
			proto += addr \|\| '';
			var url = tu.flattenStringlist( [proto].concat( path ) );
			// only need to look at last element; HTML entities are strip-proof.
			var last = url[url.length - 1];
			var trim = 0;
			if (last && last.constructor === String) {
			var strip = ',;\\.:!?';
			if (!stops.onStack( 'autourl' ).sawLParen) {
			strip += ')';
			}
			strip = new RegExp( '[' + Util.escapeRegExp(strip) + ']*$' );
			trim = strip.exec( last )[0].length;
			url[url.length - 1] = last.slice(0, last.length - trim);
			peg$currPos -= trim;
			}
			stops.pop( 'autourl' );
			return tu.flattenString( url );
			}
			/ &{ return stops.pop( 'autourl' ); }

			ipv4_address
			= $([0-9]* '.' [0-9]* '.' [0-9]* '.' [0-9]*)

			ipv6_address
			= $('[' [0-9]* ':' [0-9]* ':' [0-9]* ':' [0-9]* ':' [0-9]* ':' [0-9]* ':' [0-9]* ']')


			/**************************************************************
			* Templates, -arguments and wikilinks
			**************************************************************/

			/*
			* Precedence: template arguments win over templates. See
			* http://www.mediawiki.org/wiki/Preprocessor_ABNF#Ideal_precedence
			* 4: {{{{·}}}} → {·{{{·}}}·}
			* 5: {{{{{·}}}}} → {{·{{{·}}}·}}
			* 6: {{{{{{·}}}}}} → {{{·{{{·}}}·}}}
			* 7: {{{{{{{·}}}}}}} → {·{{{·{{{·}}}·}}}·}
			*/
			tplarg_or_template
			= & '{{{{{{{' ob:'{' tpl:tplarg_or_template eb:'}' { return [ob, tpl, eb]; }
			/ & ( '{{{' &'{{{' tplarg ) r:tplarg { return r; }
			// tplarg in template
			/ & ( '{{' &'{{{' tplarg ) r:template { return r; }
			/ tplarg
			/ template

			tplarg_or_template_or_broken
			= ( & {
			// Refuse to recurse beyond 40 levels. Default in the PHP parser
			// is $wgMaxTemplateDepth = 40;
			if ( stops.onCount('templatedepth') === undefined \|\|
			stops.onCount('templatedepth') < 40
			) {
			return stops.inc('templatedepth');
			} else {
			return false;
			}
			} r:( t:tplarg_or_template { stops.dec('templatedepth'); return t; }
			/ & { return stops.dec('templatedepth'); }
			) { return r; } ) / broken_template

			tplarg_or_template_or_bust
			= (tplarg_or_template / .)+

			broken_template
			= v:( '{{' space_or_newline+ '}}' / ( '{{{' / '}}}' / '{{' / '}}' )+ )
			{
			return [
			new TagTk('span', [ new KV('typeof', 'mw:Nowiki') ], { tsr: [peg$reportedPos, peg$reportedPos], src: text() } )]
			.concat(v, [new EndTagTk( 'span', [ new KV('typeof', 'mw:Nowiki') ], { tsr: [peg$currPos, peg$currPos] }) ]);
			}


			template
			= "{{" nl_comment_space*
			target:template_param_value
			params:(nl_comment_space* "\|"
			r:( p0:({return peg$currPos;}) v:nl_comment_space* p:({return peg$currPos;}) &"\|"
			{ return new KV( '', tu.flattenIfArray(v), [p0, p0, p0, p]); } // empty argument
			/ template_param
			) { return r; }
			)*
			nl_comment_space*
			"}}" {
			// Insert target as first positional attribute, so that it can be
			// generically expanded. The TemplateHandler then needs to shift it out
			// again.
			params.unshift( new KV(tu.flattenIfArray( target.tokens ), '', target.srcOffsets) );
			var obj = new SelfclosingTagTk( 'template', params, {tsr: [peg$reportedPos, peg$currPos], src: text()} );
			return obj;
			}

			// XXX: support template and args in target!
			//template_target
			// = $( !"}}" ([^\|\n]) )*

			tplarg
			= "{{{"
			name:template_param_value?
			params:( nl_comment_space*
			'\|' nl_comment_space*
			r:(
			&'}}}' { return new KV( '', ''); }
			/ template_param
			) { return r; }
			)*
			nl_comment_space*
			"}}}" {
			if (name) {
			params.unshift( new KV( tu.flattenIfArray(name.tokens), '', name.srcOffsets ) );
			} else {
			params.unshift( new KV( '', '') );
			}
			var obj = new SelfclosingTagTk( 'templatearg', params, {tsr: [peg$reportedPos, peg$currPos], src: text()} );
			return obj;
			}

			template_param
			= name:template_param_name
			val:(
			kEndPos:({return peg$currPos;})
			optionalSpaceToken
			"="
			vStartPos:({return peg$currPos;})
			optionalSpaceToken
			tpv:template_param_value? {
			return { kEndPos: kEndPos, vStartPos: vStartPos, value: (tpv && tpv.tokens) \|\| [] };
			}
			)? {
			if ( val !== null ) {
			if ( val.value !== null ) {
			return new KV( name, tu.flattenIfArray( val.value ), [peg$reportedPos, val.kEndPos, val.vStartPos, peg$currPos] );
			} else {
			return new KV(tu.flattenIfArray( name ), '', [peg$reportedPos, val.kEndPos, val.vStartPos, peg$currPos] );
			}
			} else {
			return new KV('', tu.flattenIfArray(name), [peg$reportedPos, peg$reportedPos, peg$reportedPos, peg$currPos] );
			}
			}
			// empty parameter
			/ & [\|}] { return new KV('', '', [peg$reportedPos, peg$reportedPos, peg$reportedPos, peg$currPos] ); }


			// FIXME: handle template args and templates in key! (or even parser functions?)
			template_param_name
			= & { return stops.push( 'equal', true ); }
			tpt:(template_param_text / &'=' { return ''; })
			{
			stops.pop( 'equal' );
			return tpt;
			}

			/ & { return stops.pop( 'equal' ); }
			//= h:( !"}}" x:([^=\|\n]) { return x } )* { return h.join(''); }

			template_param_value
			= & { stops.inc( 'nopre' ); return stops.push( 'equal', false ); }
			tpt:template_param_text
			{
			stops.dec( 'nopre' );
			stops.pop( 'equal' );
			return { tokens: tpt, srcOffsets: [peg$reportedPos, peg$currPos] };
			}
			/ & { stops.dec( 'nopre' ); return stops.pop( 'equal' ); }

			template_param_text
			= & { /*console.warn( 'tpt: ' +
			input.substr( peg$currPos - 10, 9) +
			input[peg$currPos].green +
			input.substr( peg$currPos +1, 9) ); */
			// re-enable tables within template parameters
			stops.push('table', false );
			stops.push('extlink', false);
			stops.push('pipe', true);
			return stops.inc('template');
			}
			il:(nested_block / newlineToken)+ {
			stops.pop('table');
			stops.pop('extlink');
			stops.pop('pipe');
			stops.dec('template');
			// il is guaranteed to be an array -- so, tu.flattenIfArray will
			// always return an array
			var r = tu.flattenIfArray( il );
			if ( r.length === 1 && r[0].constructor === String ) {
			r = r[0];
			}

			return r;
			}
			/ & { stops.pop('table'); stops.pop('extlink'); stops.pop('pipe'); return stops.dec('template'); }


			wikilink_content
			= lcs:( pipe startPos:({ return peg$currPos; }) lt:link_text? {
			var maybeContent = new KV( 'mw:maybeContent', lt, [startPos, peg$currPos] );
			maybeContent.vsrc = input.substring( startPos, peg$currPos );
			return maybeContent;
			} ) + {
			if ( lcs.length === 1 && lcs[0].v === null ) {
			return { content: [], pipetrick: true };
			} else {
			return { content: lcs };
			}
			}

			// TODO: handle link prefixes as in al[[Razi]]
			wikilink
			= "[["
			! url
			//target:link_target
			// XXX: disallow pipe!
			target:wikilink_preprocessor_text?
			tpos:({return peg$currPos;})
			lcontent:wikilink_content?
			"]]"
			{
			if ( lcontent === null ) {
			lcontent = { content: [] };
			}

			if ( target === null ) {
			var src = text();
			return [src];
			}

			var obj = new SelfclosingTagTk('wikilink');
			var textTokens = [];
			var hrefKV = new KV('href', target);
			hrefKV.vsrc = input.substring(peg$reportedPos + 2, tpos);
			// XXX: Point to object with path, revision and input information
			// obj.source = input;
			obj.attribs.push(hrefKV);
			obj.attribs = obj.attribs.concat( lcontent.content );
			obj.dataAttribs = {
			tsr: [peg$reportedPos, peg$currPos],
			src: text(),
			pipetrick: lcontent.pipetrick
			};
			return [obj];
			}

			// This rule is identical to the 'inline' fragment except
			// that tables are allowed inside image captions.
			link_text_fragment
			= c:((sol full_table_in_link_caption)
			/ urltext
			/ (!inline_breaks
			!pre_start
			r:( inline_element / '[' text_char+ ']' / . ) { return r; }
			)
			)+ {
			return tu.flattenStringlist( c );
			}

			link_text
			= & { return stops.inc('linkdesc'); }
			h:link_text_fragment
			// 'equal' syntaxFlag is set for links in template parameters. Consume the
			// '=' here.
			hs:( '=' link_text_fragment )?
			{
			stops.dec('linkdesc');
			if ( hs !== null ) {
			return h.concat(hs);
			} else {
			return h;
			}
			}
			/ & { return stops.dec('linkdesc'); }

			link_option
			= & { stops.push('pipe', true); return stops.inc('linkdesc'); }
			h:inline
			// 'equal' syntaxFlag is set for links in template parameters. Consume the
			// '=' here.
			hs:( '=' inline)?
			{
			stops.pop('pipe');
			stops.dec('linkdesc');
			if ( hs !== null ) {
			return h.concat(hs);
			} else {
			return h;
			}
			}
			/ & { stops.pop('pipe'); return stops.dec('linkdesc'); }

			link_end = "]]"

			/* Generic quote rule for italic and bold, further processed in a token
			* stream transformation in doQuotes. Relies on NlTk tokens being emitted
			* for each line of text to balance quotes per line.
			*
			* We are not using a simple pair rule here as we need to support mis-nested
			* bolds/italics and MediaWiki's special heuristics for apostrophes, which are
			* all not context free. */
			quote = quotes:$("''" "'"*) {
			// sequences of four or more than five quotes are assumed to start
			// with some number of plain-text apostrophes.
			var plainticks = 0;
			var result = [];
			if (quotes.length === 4) {
			plainticks = 1;
			} else if (quotes.length > 5) {
			plainticks = quotes.length - 5;
			}
			if (plainticks > 0) {
			result.push(quotes.substring(0, plainticks));
			}
			// mw-quote token Will be consumed in token transforms
			var mwq = new SelfclosingTagTk( 'mw-quote', [],
			{ tsr: [peg$reportedPos + plainticks, peg$currPos] } );
			mwq.value = quotes.substring(plainticks);
			result.push(mwq);
			return result;
			}


			/***********************************************************
			* Pre and xmlish tags
			***********************************************************/

			// Indented pre blocks differ from their non-indented (purely tag-based)
			// cousins by having their contents parsed.
			pre_indent
			= pre_indent_in_tags
			/ l:pre_indent_line
			// keep consuming indented lines unless they start a table
			ls:(s:sol
			!(space* "{\|")
			pl:pre_indent_line {
			return s.concat(pl);
			}
			)*
			{
			return l.concat(ls);
			}

			pre_tag_name =
			tag:"pre"i !tag_name_chars {
			return tag;
			}

			// An indented pre block that is surrounded with pre tags. The pre tags are
			// used directly.
			// XXX gwicke: check if the first line is not indented, and round-trip spaces;
			// possibly merge with the regular 'pre' rule.
			// FIXME: fix tag end position
			pre_indent_in_tags
			= & { return stops.inc('pre'); }
			s:spaces // XXX: capture space for round-tripping
			"<" pre_tag_name
			attribs:generic_attributes
			">"
			l:nested_block_line
			ls:(sol pre_indent_line)*
			"</" pre_tag_name ">"
			{
			stops.dec('pre');
			var ret = [ new TagTk( 'pre', attribs, { tsr: [peg$reportedPos, peg$reportedPos] } ) ];
			// ls will always be an array
			return ret.concat( l, tu.flattenIfArray( ls ), [ new EndTagTk( 'pre' ) ] );
			}
			/ & { return stops.dec('pre'); }

			// Don't recognize tabs
			pre_indent_line = " " l:nested_block_line {
			return [' '].concat(l);
			}

			/*
			* Pre blocks defined using non-indented HTML tags only parse nowiki tags and
			* html entities inside them, and convert other content to verbatim text.
			* Nowiki inside pre is not functionally needed, but supported for backwards
			* compatibility.
			*
			* TODO: add entity support!
			*/
			pre
			= & { return stops.inc('pre'); }
			"<" pre_tag_name
			attribs:generic_attributes
			space*
			endpos:(">" { return peg$currPos; })
			// MediaWiki <pre> is special in that it converts all pre content to plain
			// text.
			ts:( newlineToken
			/ (htmlentity / [^&<]+)+
			/ nowiki
			/ !("</" pre_tag_name ">") t2:(htmlentity / .) { return t2; })*
			("</" pre_tag_name ">" / eof) {
			stops.dec('pre');
			// return nowiki tags as well?

			// Emit as SelfclosingTag in order to avoid the nested pre problem in
			// the PreHandler.
			attribs.push(new KV('property', 'mw:html'));
			attribs.push(new KV('content', tu.flattenStringlist(ts)));
			return [
			new SelfclosingTagTk('pre', attribs, {
			tsr: [peg$reportedPos, peg$currPos],
			endpos: endpos
			})
			];

			}
			/ "</" pre_tag_name ">" { stops.dec('pre'); return "</pre>"; }
			// if this is still preish, emit as a string
			// necessary to work with the pre_start lookaheads
			/ p:('<' pre_tag_name) {
			stops.dec('pre');
			return tu.flattenStringlist(p);
			}
			/ & { return stops.dec('pre'); }

			/* -----------------------------------------------------------------------
			* Extension tags should be parsed with higher priority than anything else.
			* The trick we use is to strip out the content inside a matching tag-pair
			* and not tokenize it. The content, if it needs to parsed (for example,
			* for <ref>, <include> tags), is parsed in a fresh tokenizer context
			* which means any error correction that needs to happen is restricted to
			* the scope of the extension content and doesn't spill over to the higher
			* level. Ex: <math><!--foo</math>.
			*
			* This trick also lets us prevent extension content (that don't accept WT)
			* from being parsed as wikitext (Ex: <math>\frac{foo\frac{bar}}</math>)
			* We don't want the "}}" being treated as a template closing tag and closing
			* outer templates.
			* ----------------------------------------------------------------------- */

			xmlish_tag =
			t:generic_tag {
			var tagName = t.name.toLowerCase();
			var dp = t.dataAttribs;
			var isHtmlTag = Util.isHTMLElementName(tagName);
			var isInstalledExt = options.env.conf.wiki.isExtensionTag(tagName);
			var isIncludeTag = tagName === 'includeonly'
			\|\| tagName === 'noinclude'
			\|\| tagName === 'onlyinclude';

			if ( !isHtmlTag && !isInstalledExt && !isIncludeTag ) {
			return Util.newlinesToNlTks(input.substring(dp.tsr[0], dp.tsr[1]), dp.tsr[0]);
			}

			var skipLen = 0;

			// EndTagTk
			if ( t.constructor === EndTagTk \|\| isHtmlTag ) {
			return t;

			// SelfclosingTagTk
			} else if ( t.constructor === SelfclosingTagTk ) {

			dp.src = input.substring(dp.tsr[0], dp.tsr[1]);
			dp.tagWidths = [dp.tsr[1] - dp.tsr[0], 0];
			if ( !isInstalledExt ) {
			return t;
			}

			// TagTk
			} else {

			var tsr0 = dp.tsr[0];
			var endTagRE = new RegExp("^(?:.\|\n)?(</\\s" + tagName + "\\s*>)", "mi");
			var restOfInput = input.substring(tsr0);
			var tagContent = restOfInput.match(endTagRE);

			if ( !tagContent ) {
			dp.src = input.substring(dp.tsr[0], dp.tsr[1]);
			dp.tagWidths = [dp.tsr[1] - dp.tsr[0], 0];

			// We accept unclosed references tags,
			// as does the PHP parser. They will normalize
			// to self-closed in a round trip.
			if ( tagName !== 'references' \|\| !isInstalledExt ) {
			return t;
			}

			} else {
			var extSrc = tagContent[0];
			var endTagWidth = tagContent[1].length;

			if ( tagName === 'ref' ) {
			// Support 1-level nesting of <ref> tags during tokenizing.
			// <ref> tags are the exception to the rule (no nesting of ext tags)
			//
			// Expand extSrc as long as there is a <ref> tag found in the
			// extension source body.
			var s = extSrc.substring(peg$currPos - tsr0);
			while (s && s.match(new RegExp("<" + tagName + "[^<>]*>"))) {
			tagContent = restOfInput.substring(extSrc.length).match(endTagRE);
			if (tagContent) {
			s = tagContent[0];
			endTagWidth = tagContent[1].length;
			extSrc += s;
			} else {
			s = null;
			}
			}
			}

			// Extension content source
			dp.src = extSrc;
			dp.tagWidths = [peg$currPos - tsr0, endTagWidth];

			if ( !isIncludeTag && !isInstalledExt ) {
			return t;
			}

			skipLen = extSrc.length - dp.tagWidths[0] - dp.tagWidths[1];

			// If the xml-tag is a known installed (not native) extension,
			// skip the end-tag as well.
			if ( !isIncludeTag ) {
			skipLen += endTagWidth;
			}

			}

			}

			peg$currPos += skipLen;

			var ret;
			if ( !isIncludeTag ) {
			// update tsr[1] to span the start and end tags.
			dp.tsr[1] = peg$currPos;
			ret = new SelfclosingTagTk('extension', [
			new KV('typeof', 'mw:Extension'),
			new KV('name', tagName),
			new KV('about', options.env.newAboutId()),
			new KV('source', dp.src),
			new KV('options', t.attribs)
			], dp);
			} else {
			// If not a known installed extension, parse content as wikitext.
			// - include-directives: <noinclude>, <includeonly>, ...
			// - a non-html5 tag like <big>
			// Parse ext-content, strip eof, and shift tsr
			var extContent = dp.src.substring(dp.tagWidths[0], dp.src.length - dp.tagWidths[1]);
			var extContentToks = (new PegTokenizer(options.env)).tokenize(extContent);
			if (dp.tagWidths[1] > 0) {
			extContentToks = Util.stripEOFTkfromTokens(extContentToks);
			}
			Util.shiftTokenTSR(extContentToks, dp.tsr[0] + dp.tagWidths[0]);
			ret = [t].concat(extContentToks);
			}
			return ret;

			}

			/*
			* Nowiki treats anything inside it as plain text. It could thus also be
			* defined as an extension that returns its raw input text, possibly wrapped
			* in a span for round-trip information. The special treatment for nowiki in
			* pre blocks would still remain in the grammar though, so overall handling it
			* all here is cleaner.
			*/

			nowiki_tag_name =
			tag:"nowiki"i !tag_name_chars {
			return tag;
			}

			nowiki
			= "<" nowiki_tag_name space* ">"
			startTagEndPos:({return peg$currPos;})
			nc:nowiki_content
			endTagStartPos:({return peg$currPos;})
			"</" nowiki_tag_name space* ">" {
			return [
			new TagTk( 'span',
			[
			{k: 'typeof', v: 'mw:Nowiki'}
			],
			{ tsr: [peg$reportedPos, startTagEndPos] } )
			].concat( nc, [
			new EndTagTk( 'span',
			[
			{k: 'typeof', v: 'mw:Nowiki'}
			],
			{ tsr: [endTagStartPos, peg$currPos] })
			] );
			}
			// nowiki fallback: source-based round-tripping of <nowiki />.
			/ nw0:({return peg$currPos;})
			"<" nowiki_tag_name space* "/" space* ">" {
			return [
			new SelfclosingTagTk('meta',
			[new KV('typeof', 'mw:Placeholder')],
			{
			src: input.substring(nw0, peg$currPos),
			tsr: [nw0, peg$currPos]
			})
			];
			}
			// nowiki fallback: source-based round-tripping
			// of unbalanced nowiki tags that are treated as text.
			/ ! { return stops.counters.pre > 0; }
			nw0:({return peg$currPos;})
			"<" "/"? nowiki_tag_name space* "/"? space* ">" {
			var nowiki = input.substring(nw0, peg$currPos);
			return [
			new TagTk( 'span', [ new KV( 'typeof', 'mw:Placeholder' ) ], {
			src: nowiki,
			tsr: [nw0, nw0]
			} ),
			nowiki,
			new EndTagTk( 'span', [], { tsr: [peg$currPos, peg$currPos] } )
			];
			}

			// Should abort the nowiki match:
			// <pre><nowiki></pre></nowiki>
			// Should allow the </pre> in nowiki:
			// <nowiki></pre></nowiki>
			pre_break = & "</pre>" {
			return stops.counters.pre > 0 ? undefined : peg$FAILED;
			}

			nowiki_content
			= ts:( (htmlentity / [^&<]+)+
			/ "<pre" p0:optionalSpaceToken p1:[^>]* ">" p2:nowiki_content "</pre>" {
			return ["<pre"].concat(p0, p1, [">"], p2, ["</pre>"]).join('');
			}
			/ (!pre_break !("</" nowiki_tag_name space* ">") c:(htmlentity / .) {
			return c;
			})
			)* {
			// return nowiki tags as well?
			return tu.flattenStringlist(ts);
			}

			/* Generic XML-like tags
			*
			* These also cover extensions (including Cite), which will hook into the
			* token stream for further processing. The content of extension tags is
			* parsed as regular inline, but the source positions of the tag are added
			* to allow reconstructing the unparsed text from the input. */

			// See http://www.w3.org/TR/html5/syntax.html#tag-open-state and
			// following paragraphs.
			tag_name_chars = [^\t\n\v />\0]
			tag_name = $([A-Za-z] tag_name_chars*)

			generic_tag
			= "<"
			end:"/"? name:tag_name
			attribs:generic_newline_attribute*
			space_or_newline* // No need to preserve this -- canonicalize on RT via dirty diff
			selfclose:"/"?
			bad_ws:space* // No need to preserve this -- canonicalize on RT via dirty diff
			">" {
			var lcName = name.toLowerCase();
			var isVoidElt = Util.isVoidElement( lcName ) ? true : null;
			// Support </br>
			var broken = false;
			if (lcName === 'br' && end) {
			broken = true;
			end = null;
			}

			var res = tu.buildXMLTag(name, lcName, attribs, end, selfclose \|\| isVoidElt, [peg$reportedPos, peg$currPos]);

			// change up data-attribs in one scenario
			// void-elts that aren't self-closed ==> useful for accurate RT-ing
			if (selfclose === null && isVoidElt) {
			res.dataAttribs.selfClose = undefined;
			res.dataAttribs.noClose = true;
			}
			if (broken \|\| bad_ws.length > 0) {
			res.dataAttribs.brokenHTMLTag = true;
			}
			return res;
			}

			could_be_attribute =
			// quick sanity check before expensive attribute_preprocessor_text_line
			// rule. Also try to parse on [\|!+;] for now which seem to be common
			// syntax errors in production that hidden by the PHP parser (by stripping
			// the 'attributes').
			space* ([a-zA-Z\|!+;] /
			// Crude heuristic that just excludes attribute-less row-syntax
			// table cells like this with simple (pipe-less) values: \|a\|\|b\|\|c
			[^\'\"\n\|]+ '\|' [^\|\n] /
			// Possibly a templated attribute
			'{{' [^}]+ '}' /
			// comment or noincludes
			'<' ('!--' / 'noinclude' / 'onlyinclude' / 'includeonly'))

			// A generic attribute that can span multiple lines.
			generic_newline_attribute
			= s:space_or_newline*
			namePos0:({return peg$currPos;})
			name:generic_attribute_name
			namePos:({return peg$currPos;})
			valueData:( space_or_newline*
			v:generic_attribute_newline_value { return v; })?
			{
			var res;

			// Encapsulate protected attributes.
			if ( typeof name === "string" ) {
			name = name.replace(
			/^(about\|data-parsoid.\|data-x.\|property\|rel\|typeof)$/i,
			"data-x-$1" );
			}

			if ( valueData !== null ) {
			var value = valueData.value;
			res = new KV( name, value );
			res.vsrc = valueData.valueSrc;
			} else {
			res = new KV( name, '' );
			}
			if ( Array.isArray(name) ) {
			res.ksrc = input.substring( namePos0, namePos );
			}
			return res;
			}

			// A single-line attribute.
			generic_attribute
			= s:optionalSpaceToken
			namePos0:({return peg$currPos;})
			name:generic_attribute_name
			namePos:({return peg$currPos;})
			valueData:(optionalSpaceToken
			v:generic_attribute_value { return v; })?
			{
			// FIXME: name might just be a template, which can expand to a key-value
			// pair later. We'll need to handle that in the AttributeTransformManager.
			var res;
			if ( valueData !== null ) {
			var value = valueData.value;
			res = new KV( name, value );
			res.vsrc = valueData.valueSrc;
			} else {
			res = new KV( name, '' );
			}
			if ( Array.isArray(name) ) {
			res.ksrc = input.substring( namePos0, namePos );
			}
			return res;
			}

			// ( Replaced by generic_attribute_name for template / parameter support. )
			//// http://www.w3.org/TR/html5/syntax.html#attributes-0, and we also
			//// disallow newlines, \| and {.
			//generic_attribute_plain_name
			// = n:[^ \t\0/"'>=\n\|{]+ {
			// return n.join('');
			// }

			// Also eat these chars in a wikitext table or tr attribute name. They are
			// normally not matched by the generic_attribute_name.
			broken_table_attribute_name_char = c:[ \t\0/=>"'\[] { return new KV(c, ''); }

			// The arrangement of chars is to emphasize the split between what's disallowed
			// by html5 and what's necessary to give directive a chance.
			generic_attribute_name
			= r:( ts:[^ \t\0\n\r/=>"'!<&\[\]\|{}\-]+ { return ts.join(''); }
			/ ! inline_breaks
			! '/>'
			// /=>"' is the html5 attribute name set we do not want.
			// \[ is to avoid eating links. (see: BUG 553: link with two variables in a piped link)
			t:( directive / !( space_or_newline / [\[/=>"'] ) c:. { return c; }
			) { return t; }
			)+ {
			return tu.flattenString( r );
			}

			// A generic attribute, possibly spanning multiple lines.
			generic_attribute_newline_value
			= "=" v:xml_att_value? {
			return v === null ? [] : v;
			}

			// A generic but single-line attribute.
			generic_attribute_value
			= "=" v:att_value? {
			return v === null ? [] : v;
			}

			// Attribute value, quoted variants can span multiple lines.
			xml_att_value
			= space_or_newline* "'" r:(valPos1:({return peg$currPos;}) t1:attribute_preprocessor_text_single? valPos2:({return peg$currPos;}) "'"
			{ return tu.getAttributeValueAndSource(input, t1, valPos1, valPos2); }
			// Missing end quote: accept \| and > look-ahead as heuristic
			/ valPos1:({return peg$currPos;}) t2:attribute_preprocessor_text_single_broken? valPos2:({return peg$currPos;}) &[\|>]
			{ return tu.getAttributeValueAndSource(input, t2, valPos1, valPos2); } )
			{ return r; }
			/ space_or_newline* '"' r:(valPos1:({return peg$currPos;}) t1:attribute_preprocessor_text_double? valPos2:({return peg$currPos;}) '"'
			{ return tu.getAttributeValueAndSource(input, t1, valPos1, valPos2); }
			// Missing end quote: accept \| and > look-ahead as heuristic
			/ valPos1:({return peg$currPos;}) t2:attribute_preprocessor_text_double_broken? valPos2:({return peg$currPos;}) &[\|>]
			{ return tu.getAttributeValueAndSource(input, t2, valPos1, valPos2); } )
			{ return r; }
			/ space_or_newline* valPos1:({return peg$currPos;}) t:attribute_preprocessor_text? !"=" valPos2:({return peg$currPos;})
			{ if ( t === null ) {
			t = "";
			// Reset the current parse position in order to reparse any
			// captured spaces, which are needed to separate attributes.
			peg$currPos = peg$reportedPos;
			}
			return tu.getAttributeValueAndSource(input, t, valPos1, valPos2); }

			// Attribute value, restricted to a single line.
			att_value
			= space* "'" r:(valPos1:({return peg$currPos;}) t1:attribute_preprocessor_text_single_line? valPos2:({return peg$currPos;}) "'"
			{ return tu.getAttributeValueAndSource(input, t1, valPos1, valPos2); }
			// Missing end quote: accept \| and > look-ahead as heuristic
			/ valPos1:({return peg$currPos;}) t2:attribute_preprocessor_text_single_line_broken? valPos2:({return peg$currPos;}) &[\|>\n]
			{ return tu.getAttributeValueAndSource(input, t2, valPos1, valPos2); } )
			{ return r; }
			/ space* '"' r:(valPos1:({return peg$currPos;}) t1:attribute_preprocessor_text_double_line? valPos2:({return peg$currPos;}) '"'
			{ return tu.getAttributeValueAndSource(input, t1, valPos1, valPos2); }
			// Missing end quote: accept \| and > look-ahead as heuristic
			/ valPos1:({return peg$currPos;}) t2:attribute_preprocessor_text_double_line_broken? valPos2:({return peg$currPos;}) &[\|>\n]
			{ return tu.getAttributeValueAndSource(input, t2, valPos1, valPos2); } )
			{ return r; }
			/ space* valPos1:({return peg$currPos;}) t:attribute_preprocessor_text_line? !"=" valPos2:({return peg$currPos;})
			{ if ( t === null ) {
			t = "";
			// Reset the current parse position in order to reparse any
			// captured spaces, which are needed to separate attributes.
			peg$currPos = peg$reportedPos;
			}
			return tu.getAttributeValueAndSource(input, t ? t : "", valPos1, valPos2); }

			/*
			* A variant of generic_tag, but also checks if the tag name is a block-level
			* tag as defined in
			* http://www.w3.org/TR/html5/syntax.html#tag-open-state and
			* following paragraphs.
			*/
			block_tag
			= "<" end:"/"?
			name:tag_name
			attribs:generic_newline_attribute*
			space_or_newline*
			selfclose:"/"?
			">" {
			var lcName = name.toLowerCase();
			if (lcName !== "pre" && lcName !== "hr" &&
			constants.HTML.BlockTags.has(name.toUpperCase())
			) {
			return [tu.buildXMLTag(name, lcName, attribs, end, selfclose, [peg$reportedPos, peg$currPos])];
			} else {
			// abort match if tag is not block-level
			peg$currPos = peg$reportedPos;
			return peg$FAILED;
			}
			}


			/*********************************************************
			* Lists
			*********************************************************/
			list_item = dtdd / hacky_dl_uses / li

			li = bullets:list_char+
			c:nested_block_line
			&eolf
			{
			if ( c === null ) {
			c = [];
			}
			// Leave bullets as an array -- list handler expects this
			var li = new TagTk( 'listItem', [], { tsr: [peg$reportedPos, peg$reportedPos + bullets.length] } );
			li.bullets = bullets;
			return [ li, c ];
			}

			/*
			* This rule is required to support wikitext of this form
			* ::{\|border="1"\|foo\|bar\|baz\|}
			* where the leading colons are used to indent the entire table.
			* This hack was added back in 2006 in commit
			* a0746946312b0f1eda30a2c793f5f7052e8e5f3a based on a patch by Carl
			* Fürstenberg.
			*/
			hacky_dl_uses = bullets:":"+
			tbl:(table_lines (sol table_lines)*)
			s:space* // Do we really need to RT this?
			&comment_space_eolf
			{
			// Leave bullets as an array -- list handler expects this
			var li = new TagTk( 'listItem', [], { tsr: [peg$reportedPos, peg$reportedPos + bullets.length] } );
			li.bullets = bullets;
			return tu.flattenIfArray([li, tbl \|\| [], s \|\| []]);
			}

			dtdd
			= bullets:(!(";" !list_char) lc:list_char { return lc; })*
			";"
			& {return stops.inc('colon');}
			c:nested_block_line
			cpos:(":" { return peg$currPos; })
			// Fortunately dtdds cannot be nested, so we can simply set the flag
			// back to 0 to disable it.
			& { stops.counters.colon = 0; return true;}
			d:nested_block_line?
			&eolf {
			// Leave bullets as an array -- list handler expects this
			// TSR: +1 for the leading ";"
			var numBullets = bullets.length + 1;
			var li1 = new TagTk( 'listItem', [], { tsr: [peg$reportedPos, peg$reportedPos + numBullets] } );
			li1.bullets = bullets.slice();
			li1.bullets.push(";");
			// TSR: -1 for the intermediate ":"
			var li2 = new TagTk( 'listItem', [], { tsr: [cpos - 1, cpos], stx: 'row' } );
			li2.bullets = bullets.slice();
			li2.bullets.push(":");

			return [ li1 ].concat( c, [ li2 ], d \|\| [] );
			}
			// Fall-back case to clear the colon flag
			/ & { stops.counters.colon = 0; return false; }


			list_char = [*#:;]



			/******************************************************************************
			* Tables
			* ------
			* Table rules are geared to support independent parsing of fragments in
			* templates (the common table start / row / table end use case). The tokens
			* produced by these fragments then match up to a table while building the
			* DOM tree. For similar reasons, table rows do not emit explicit end tag
			* tokens.
			*
			* The separate table_lines rule is faster than moving those rules
			* directly to block_lines.
			*
			* Notes about the full_table_in_link_caption rule
			* -----------------------------------------------------
			* However, for link-tables, we have introduced a stricter parse wherein
			* we require table-start and table-end tags to not come from a template.
			* In addition, this new rule doesn't accept fosterable-content in
			* the table unlike the more lax (sol table_lines)+ rule.
			*
			* This is the best we can do at this time since we cannot distinguish
			* between table rows and image options entirely in the tokenizer.
			*
			* Consider the following examples:
			*
			* Example 1:
			*
			* [[Image:Foo.jpg\|left\|30px\|Example 1
			* {{This-template-returns-a-table-start-tag}}
			* \|foo
			* {{This-template-returns-a-table-end-tag}}
			* ]]
			*
			* Example 2:
			*
			* [[Image:Foo.jpg\|left\|30px\|Example 1
			* {{echo\|a}}
			* \|foo
			* {{echo\|b}}
			* ]]
			*
			* So, we cannot know a priori (without preprocessing or fully expanding
			* all templates) if "\|foo" in the two examples is a table cell or an image
			* option. This is a limitation of our tokenizer-based approach compared to
			* the preprocessing-based approach of the PHP parser.
			*
			* Given this limitation, we are okay forcing a full-table context in
			* link captions (if necessary, we can relax the fosterable-content requirement
			* but that is broken wikitext anyway, so we can force that edge-case wikitext
			* to get fixed by rejecting it).
			******************************************************************************/

			full_table_in_link_caption
			= (! inline_breaks / & '{{!}}' )
			r:(
			& { return stops.push('table', true); }
			tbl:(
			table_start_tag optionalNewlines
			(sol table_content_line optionalNewlines)*
			sol table_end_tag)
			{
			stops.pop('table');
			return tbl;
			}
			/ & { return stops.pop('table'); }
			) { return r; }

			table_lines
			= (! inline_breaks / & '{{!}}' )
			r:(
			& { return stops.push('table', true); }
			tl:table_line
			nls:optionalNewlines
			{
			stops.pop('table');
			return tl.concat(nls);
			}
			/ & { return stops.pop('table'); }
			) { return r; }

			// This rule assumes start-of-line position!
			table_line
			= table_start_tag
			/ table_content_line
			/ table_end_tag

			table_content_line = (space / comment)* (
			table_heading_tags
			/ table_row_tag
			/ table_data_tags
			/ table_caption_tag
			)

			table_start_tag
			= sc:(space / comment)* startPos:({ return peg$currPos; }) b:"{" p:pipe
			// ok to normalize away stray \|} on rt (see bug 57360)
			& { return stops.push('table', false); }
			ta:(generic_attribute / broken_table_attribute_name_char)*
			tsEndPos:({stops.pop('table'); return peg$currPos;})
			{
			var coms = tu.popComments( ta );
			if ( coms ) {
			tsEndPos = coms.commentStartPos;
			}

			var da = { tsr: [startPos, tsEndPos] };
			if ( p !== "\|" ) {
			// Variation from default
			da.startTagSrc = b + p;
			}

			sc.push( new TagTk( 'table', ta, da ) );
			if ( coms ) {
			sc = sc.concat( coms.buf );
			}
			return sc;
			}

			table_caption_tag
			// avoid recursion via nested_block_in_table
			= ! { return stops.onStack('tableDataBlock'); }
			p:pipe "+"
			args:single_cell_table_args?
			tagEndPos:({return peg$currPos;})
			c:nested_block_in_table* {
			return tu.buildTableTokens("caption", "\|+", args, [peg$reportedPos, tagEndPos], peg$currPos, c)
			.concat([new EndTagTk('caption')]);
			}

			table_row_tag
			= // avoid recursion via nested_block_in_table
			! { return stops.onStack('tableDataBlock'); }
			p:pipe dashes:$"-"+
			& { return stops.push('table', false); }
			a:(generic_attribute / broken_table_attribute_name_char)*
			tagEndPos:({stops.pop('table'); return peg$currPos;})
			// handle tables with missing table cells after a row
			td:implicit_table_data_tag?
			{
			var coms = tu.popComments( a );
			if ( coms ) {
			tagEndPos = coms.commentStartPos;
			}

			var da = {
			tsr: [ peg$reportedPos, tagEndPos ],
			startTagSrc: p + dashes
			};

			// We rely on our tree builder to close the row as needed. This is
			// needed to support building tables from fragment templates with
			// individual cells or rows.
			var trToken = new TagTk( 'tr', a, da );

			var res = [ trToken ];
			if ( coms ) {
			res = res.concat( coms.buf );
			}
			if ( td ) {
			res = res.concat( td );
			}
			return res;
			}

			tds
			= ( pp:( pipe_pipe / p:pipe & row_syntax_table_args { return p; } )
			tdt:table_data_tag {
			var da = tdt[0].dataAttribs;
			da.stx_v = "row";
			da.tsr[0] = da.tsr[0] - pp.length; // include "\|\|"
			if (pp !== "\|\|" \|\| (da.startTagSrc && da.startTagSrc !== pp)) {
			// Variation from default
			da.startTagSrc = pp + (da.startTagSrc ? da.startTagSrc.substring(1) : '');
			}
			return tdt;
			}
			)*

			table_data_tags
			// avoid recursion via nested_block_in_table
			= ! { return stops.onStack('tableDataBlock'); }
			p:pipe
			![+-] td:table_data_tag
			tagEndPos:({return peg$currPos;})
			tds:tds {
			var da = td[0].dataAttribs;
			da.tsr[0] = da.tsr[0] - p.length; // include "\|"
			if (p !== "\|") {
			// Variation from default
			da.startTagSrc = p;
			}
			return td.concat(tds);
			}

			implicit_table_data_tag
			= & sol // Implicit table data tag added only when content starts on a newline
			!( sol+ (pipe / [!+-]) )
			! "}"
			tagEndPos:({return peg$currPos;})
			b:nested_block+
			tds:tds {
			b = tu.flattenIfArray(b);
			var nlTok = b.shift();
			var td = tu.buildTableTokens("td", "\|", '', [nlTok.dataAttribs.tsr[1], tagEndPos], peg$currPos, b);
			td[0].dataAttribs.autoInsertedStart = true;
			td[0].dataAttribs.autoInsertedEnd = true;
			return [ nlTok ].concat( td, tds );
			}

			table_data_tag
			= ! "}"
			arg:row_syntax_table_args?
			// use inline_breaks to break on tr etc
			tagEndPos:({return peg$currPos;})
			td:nested_block_in_table*
			{
			return tu.buildTableTokens("td", "\|", arg, [peg$reportedPos, tagEndPos], peg$currPos, td);
			}

			table_heading_tags
			= "!"
			th:table_heading_tag
			ths:( pp:("!!" / pipe_pipe) tht:table_heading_tag {
			var da = tht[0].dataAttribs;
			da.stx_v = 'row';
			da.tsr[0] = da.tsr[0] - pp.length; // include "!!" or "\|\|"

			if (pp !== "!!" \|\| (da.startTagSrc && da.startTagSrc !== pp)) {
			// Variation from default
			da.startTagSrc = pp + (da.startTagSrc ? da.startTagSrc.substring(1) : '');
			}
			return tht;
			}
			)* {
			th[0].dataAttribs.tsr[0]--; // include "!"
			return th.concat(ths);
			}

			table_heading_tag
			= & { return stops.push('th', true); }
			arg:row_syntax_table_args?
			tagEndPos:({return peg$currPos;})
			c:nested_block_in_table* {
			stops.pop('th');
			return tu.buildTableTokens("th", "!", arg, [peg$reportedPos, tagEndPos], peg$currPos, c);
			}
			/ & { return stops.pop('th'); }

			table_end_tag
			= sc:(space / comment)* startPos:({ return peg$currPos; }) p:pipe b:"}" {
			var tblEnd = new EndTagTk( 'table', [], { tsr: [startPos, peg$currPos] } );
			if (p !== "\|") {
			// p+"<brace-char>" is triggering some bug in pegJS
			// I cannot even use that expression in the comment!
			tblEnd.dataAttribs.endTagSrc = p + b;
			}
			return sc.concat([tblEnd]);
			}

			/**
			* Table parameters separated from the content by a single pipe. Matches even
			* if there are more pipes following.
			*/
			single_cell_table_args
			= & { return stops.push('pipe', true); }
			as:generic_attributes s:space* p:pipe {
			stops.pop('pipe');
			return [as, s, p];
			}
			/ & { return stops.pop('pipe'); }

			/**
			* Table parameters separated from the content by a single pipe. Does not
			* match if followed by double pipe (row-based syntax).
			*/
			row_syntax_table_args
			= & { return stops.inc('tableCellArg'); }
			& could_be_attribute
			as:generic_attributes s:space* p:pipe !pipe {
			stops.dec('tableCellArg');
			return [as, s, p];
			}
			/ & { return stops.dec('tableCellArg'); }


			/*******************************************************************
			* Text variants and other general rules
			*******************************************************************/

			/* All chars that cannot start syntactic structures in the middle of a line
			* XXX: ] and other end delimiters should probably only be activated inside
			* structures to avoid unnecessarily leaving the text rule on plain
			* content.
			*
			* TODO: Much of this is should really be context-dependent (syntactic
			* flags). The wikilink_preprocessor_text rule is an example where
			* text_char is not quite right and had to be augmented. Try to minimize /
			* clarify this carefully!
			*/

			text_char = [^-'<~[{\n\r:;\]}\|!=]

			/* Legend
			* ' quotes (italic/bold)
			* < start of xmlish_tag
			* ~ signatures/dates
			* [ start of links
			* { start of parser functions, transclusion and template args
			* \n all sort of block-level markup at start of line
			* \r ditto
			* h http(s) urls
			* n nntp(s) urls
			* m mailto urls
			* I start of ISBN 10/13 auto links
			* P start of PMID auto links
			* R start of RFC auto links
			*
			* _ behavior switches (e.g., '__NOTOC__') (XXX: not URL related)
			* ! and \| table cell delimiters, might be better to specialize those
			* = headings - also specialize those!
			*
			* The following chars are also included for now, but only apply in some
			* contexts and should probably be enabled only in those:
			* : separate definition in ; term : definition
			* ] end of link
			* } end of parser func/transclusion/template arg
			* - start of lang_variant -{ ... }-
			* ; separator in lang_variant
			*/

			urltext = ( $[^-'<~[{\n\pPrRfFgGhHiImMnNsStTwW_\|!:;\]} &=]+
			/ & [/fFgGhHiImMnNsStTwWIPR] al:autolink { return al; }
			/ & "&" he:htmlentity { return he; }
			// Convert trailing space into
			// XXX: This should be moved to a serializer
			// This is a hack to force a whitespace display before the colon
			/ ' ' & ':' {
			return [
			new TagTk( 'span', [ new KV( 'typeof', 'mw:DisplaySpace mw:Placeholder' ) ], { src: ' ', tsr: [peg$reportedPos, peg$reportedPos], isDisplayHack: true } ),
			"\u00a0",
			new EndTagTk( 'span', [], { tsr: [peg$currPos, peg$currPos], isDisplayHack: true } )
			];
			}
			/ & ('__') bs:behavior_switch { return bs; }
			// About 96% of text_char calls originate here.
			// pegjs 0.8 inlines this simple rule automatically.
			/ text_char )+

			/*
			'//', // for protocol-relative URLs, but not in text!
			'ftp://',
			'git://',
			'gopher://',
			'http://',
			'https://',
			'irc://',
			'ircs://', // @bug 28503
			'mailto:',
			'mms://',
			'news:',
			'nntp://', // @bug 3808 RFC 1738
			'svn://',
			'telnet://', // Well if we're going to support the above.. -ævar
			'worldwind://',
			*/

			// Old version
			//text = t:[A-Za-z0-9,._ "?!\t-]+ { return t.join('') }

			htmlentity = m:$("&" [#0-9a-zA-Z]+ ";") {
			var cc = Util.decodeEntities(m);
			// if this is an invalid entity, don't tag it with 'mw:Entity'
			if (cc.length > 2 /* decoded entity would be 1 or 2 UTF-16 characters */) {
			return cc;
			}
			return [
			new TagTk('span', [new KV('typeof', 'mw:Entity')], { src: m, srcContent: cc, tsr: [peg$reportedPos, peg$reportedPos] } ),
			cc,
			new EndTagTk('span', [], { tsr: [peg$currPos, peg$currPos] })
			];
			}

			spaces
			= $[ \t]+

			space = [ \t]

			optionalSpaceToken
			= s:$space* {
			if ( s.length ) {
			return [s];
			} else {
			return [];
			}
			}

			/* This rule corresponds to \s in the PHP preg_* functions,
			* which is used frequently in the PHP parser. The inclusion of
			* form feed (but not other whitespace, like vertical tab) is a quirk
			* of Perl, which PHP inherited via the PCRE (Perl-Compatible Regular
			* Expressions) library.
			*/
			space_or_newline
			= [ \t\n\r\x0c]

			/* This rule corresponds to \b in the PHP preg_* functions,
			* after a word character. That is, it's a zero-width lookahead that
			* the next character is not a word character.
			*/
			end_of_word
			= eof / ! [A-Za-z0-9_] { return ''; }

			// Extra newlines followed by at least another newline. Usually used to
			// compress surplus newlines into a meta tag, so that they don't trigger
			// paragraphs.
			optionalNewlines
			= spc:$([\n\r\t ] &([\n\r]))* {
			if ( spc.length ) {
			return [spc];
			} else {
			return [];
			}
			}

			sol = (empty_line_with_comments / sol_prefix) (comment / (
			( & { return stops.push("sol_il", true); }
			i:include_limits
			& { stops.pop("sol_il"); return true; }
			) { return i; }
			/ & { return stops.pop("sol_il"); }
			))*

			sol_prefix
			= newlineToken
			/ & {
			// Use saved sol-state only at start of input
			// If we have saved state of not being in sol posn, fail the rule
			// NOTE: Explicitly check for 'false' and not a falsy value
			return peg$currPos === 0 && options.pegTokenizer.savedSOL !== false;
			} { return []; }

			empty_line_with_comments
			= sp:sol_prefix p:({return peg$currPos;}) c:(space* comment (space / comment)* newline)+ {
			return [
			sp,
			new SelfclosingTagTk("meta", [new KV('typeof', 'mw:EmptyLine')], {
			tokens: tu.flattenIfArray(c),
			tsr: [p, peg$currPos]
			})
			];
			}

			comment_space = comment / space
			nl_comment_space = newline / comment_space

			/**
			* noinclude / includeonly / onlyinclude rules. These are normally
			* handled by the generic_tag rule, except where generic tags are not
			* allowed- for example in directives, which are allowed in various attribute
			* names and -values.
			*
			* Example test case:
			* {\|
			* \|-<includeonly>
			* foo
			* </includeonly>
			* \|Hello
			* \|}
			*/

			include_limits =
			"<" c:"/"? name:$[oyinclude]i+ space_or_newline* ">" {
			var incl = name.toLowerCase();

			if ( incl !== "noinclude" && incl !== "onlyinclude" && incl !== "includeonly" ) {
			peg$currPos = peg$reportedPos;
			return peg$FAILED;
			}

			var dp = { tsr: [peg$reportedPos, peg$currPos] };

			// Record variant since tag is not in normalized lower case
			if ( name !== incl ) {
			dp.srcTagName = name;
			}

			// End tag only
			if ( c ) {
			return new EndTagTk(name, [], dp);
			}

			var restOfInput = input.substring(peg$currPos);
			var tagContent = restOfInput.match(new RegExp("^([^]?)(?:</\\s" + incl + "\\s*>)", "m"));

			// Start tag only
			if ( !tagContent \|\| !tagContent[1] ) {
			return new TagTk(name, [], dp);
			}

			// Get the content
			var inclContent = tagContent[1];

			// Preserve SOL where necessary (for onlyinclude and noinclude)
			// Note that this only works because we encounter <include> tags in
			// the toplevel content and we rely on the php preprocessor to expand
			// templates, so we shouldn't ever be tokenizing inInclude.
			// Last line should be empty (except for comments)
			if ( incl !== "includeonly" && stops.onStack("sol_il") ) {
			var last = inclContent.split("\n");
			if ( !/^(<!--([^-]\|-(?!->))-->)$/.test(last[last.length - 1]) ) {
			peg$currPos = peg$reportedPos;
			return peg$FAILED;
			}
			}

			// Tokenize include content in a new tokenizer
			var inclContentToks = (new PegTokenizer(options.env)).tokenize(inclContent);
			inclContentToks = Util.stripEOFTkfromTokens(inclContentToks);

			// Shift tsr
			Util.shiftTokenTSR(inclContentToks, peg$currPos);

			// Skip past content
			peg$currPos += inclContent.length;

			return [new TagTk(name, [], dp)].concat(inclContentToks);
			}

			// Start of file
			sof = & { return peg$currPos === 0 && !options.startOffset && !options.pipelineOffset; }

			// End of file
			eof = & { return peg$currPos === input.length; }

			newline = '\n' / '\r\n'

			newlineToken = newline { return [new NlTk([peg$reportedPos, peg$currPos])]; }

			eolf = newline / eof

			comment_space_eolf = (space+ / comment)* (newline / eof)

			// 'Preprocessor' directive- higher-level things that can occur in otherwise
			// plain-text content.
			directive
			= comment
			/ nowiki
			/ tplarg_or_template
			/ & "&" e:htmlentity { return e; }
			/ include_limits

			wikilink_preprocessor_text
			= r:( t:$[^<[{\n\r\t\|!\]}{ &\-]+
			// XXX gwicke: any more chars we need to allow here?
			/ !inline_breaks wr:( directive / $( !"]]" ( text_char / [!<\-] ) ) )
			{ return wr; }
			)+ {
			return tu.flattenStringlist( r );
			}

			extlink_preprocessor_text
			// added special separator character class inline: separates url from
			// description / text
			= r:( $[^'<~[{\n\r\|!\]}\t&="' \u00A0\u1680\u180E\u2000-\u200A\u202F\u205F\u3000]+
			/ !inline_breaks s:( directive / no_punctuation_char ) { return s; }
			/// urlencoded_char
			// !inline_breaks no_punctuation_char
			/ $([.:,] !(space / eolf))
			/ $(['] !(['])) // single quotes are ok, double quotes are bad
			/ [&%\|{] )+ {
			return tu.flattenString( r );
			}

			// Attribute values with preprocessor support
			attribute_preprocessor_text
			= r:( $(!inline_breaks [^=<>{}\n\r&'"\t/ \-])+
			/ !inline_breaks
			! '/>'
			s:(
			directive
			/ [&%/{}\-]
			) { return s; }
			)+
			{
			return tu.flattenString( r );
			}

			attribute_preprocessor_text_single
			= r:( $[^{}&'<\-]+
			/ !inline_breaks s:(
			directive
			/ [{}&<\-] ) { return s; }
			)*
			{
			return tu.flattenString( r );
			}

			attribute_preprocessor_text_single_broken
			= r:( $[^{}&'<>\|\-]+
			/ !inline_breaks s:(
			directive
			/ [{}&<\-] ) { return s; }
			)*
			{
			return tu.flattenString( r );
			}

			attribute_preprocessor_text_double
			= r:( $[^{}&"<\-]+
			/ !inline_breaks s:(
			directive
			/ [{}&<\-] ) { return s; }
			)*
			{
			return tu.flattenString( r );
			}

			attribute_preprocessor_text_double_broken
			= r:( $[^{}&"<>\|\-]+
			/ !inline_breaks s:(
			directive
			/ [{}&<\-] ) { return s; }
			)*
			{
			return tu.flattenString( r );
			}

			// Variants with the entire attribute on a single line
			attribute_preprocessor_text_line
			= r:( $[^=<>\n\r&'"\t \[\]\|{}/!\-]+
			/ !inline_breaks
			! '/>'
			t:(
			directive
			// Eat insane tags-inside-attributes. Example:
			// <hiddentext>generated with.. </hiddentext>
			/ &generic_tag nb:nested_block_line { return nb; }
			/ !(space_or_newline / [\[=>]) c:. {
			return c;
			}
			) { return t; }
			)+
			{
			return tu.flattenString( r );
			}

			attribute_preprocessor_text_single_line
			= r:( $[^{}&'<\n\-]+
			/ !inline_breaks s:(
			directive
			/ $( ![\r\n] [{}&<\-] ) ) { return s; }
			)* {
			return tu.flattenString( r );
			}

			attribute_preprocessor_text_single_line_broken
			= r:( $[^{}&'<>\|!\n\-]+
			/ !inline_breaks s:(
			directive
			/ $( ![\r\n] q:[{}&<\-] ) ) { return s; }
			)* {
			return tu.flattenString( r );
			}

			attribute_preprocessor_text_double_line
			= r:( $[^{}&"<\n\-]+
			/ !inline_breaks s:(
			directive
			/ $( ![\r\n] [{}&<\-] ) ) { return s; }
			)* {
			return tu.flattenString( r );
			}

			attribute_preprocessor_text_double_line_broken
			= r:( $[^{}&"<>\|!\n\-]+
			/ !inline_breaks s:(
			directive
			/ $( ![\r\n] q:[{}&<\-] ) ) { return s; }
			)* {
			return tu.flattenString( r );
			}

			// Special-case support for those pipe templates
			pipe = "\|" / "{{!}}"

			end_pipe = $( "\|" ! "\|" / "{{!}}" ! "{{!}}" )

			// SSS FIXME: what about \|{{!}} and {{!}}\|
			pipe_pipe = "\|\|" / "{{!}}{{!}}"

			// Similar, for tables..
			exclam = "!" / "{{;}}"

			/* Tabs do not mix well with the hybrid rule syntax */
			/* vim: set filetype=javascript expandtab ts=4 sw=4 cindent : */

rGPAR/lib/pegTokenizer.pegjs.txt3df9b319977fmaster

pegTokenizer.pegjs.txtView Options

rGPAR/lib/pegTokenizer.pegjs.txt
3df9b319977fmaster

pegTokenizer.pegjs.txt
View Options