Diffusion parsoid (master) Browse

rGPAR/lib/mediawiki.tokenizer.peg.js
3df9b319977fmaster

mediawiki.tokenizer.peg.js
View Options

			/**
			* Tokenizer for wikitext, using PEG.js and a separate PEG grammar file
			* (pegTokenizer.pegjs.txt)
			*
			* Use along with a HTML5TreeBuilder and the DOMPostProcessor(s) for HTML
			* output.
			*
			*/
			'use strict';
			require('./core-upgrade.js');

			var PEG = require('pegjs');
			var path = require('path');
			var fs = require('fs');
			var events = require('events');
			var util = require('util');
			var JSUtils = require('./jsutils.js').JSUtils;


			// allow dumping compiled tokenizer to disk, for linting & debugging
			var PARSOID_DUMP_TOKENIZER = process.env.PARSOID_DUMP_TOKENIZER \|\| false;

			/**
			* Includes passed to the tokenizer, so that it does not need to require those
			* on each call. They are available as pegArgs.pegIncludes, and are unpacked
			* in the head of pegTokenizer.pegjs.txt.
			*/
			var pegIncludes = {
			DOMUtils: require('./mediawiki.DOMUtils.js').DOMUtils,
			Util: require('./mediawiki.Util.js').Util,
			defines: require('./mediawiki.parser.defines.js'),
			tu: require('./mediawiki.tokenizer.utils.js'),
			constants: require('./mediawiki.wikitext.constants.js').WikitextConstants,
			// defined below to satisfy JSHint
			PegTokenizer: null,
			};

			function PegTokenizer( env, options ) {
			events.EventEmitter.call(this);
			this.env = env;
			this.options = options \|\| {};
			this.offsets = {};
			}

			pegIncludes.PegTokenizer = PegTokenizer;

			// Inherit from EventEmitter
			util.inherits(PegTokenizer, events.EventEmitter);

			PegTokenizer.prototype.src = '';

			PegTokenizer.prototype.initTokenizer = function() {

			// Construct a singleton static tokenizer.
			var pegSrcPath = path.join( __dirname, 'pegTokenizer.pegjs.txt' );
			this.src = fs.readFileSync( pegSrcPath, 'utf8' );

			var tokenizerSource = PEG.buildParser(this.src, {
			cache: true,
			trackLineAndColumn: false,
			output: "source",
			allowedStartRules: [
			"start",
			"toplevelblock",
			"table_start_tag",
			"url",
			"single_cell_table_args",
			"generic_attributes",
			"tplarg_or_template_or_bust"
			]
			});

			// Bucket the cache by offset so that clearing it is easier.
			// Include the stops key in the cache key;
			tokenizerSource = tokenizerSource
			.replace(/ var key = peg\$currPos \* \d+ \+ (\d+),\n cached = peg\$cache\[key\];/g,
			" var key = \"$1\" + stops.key;\n" +
			" var bucket = peg$currPos;\n" +
			" if ( !peg$cache[bucket] ) { peg$cache[bucket] = {}; }\n" +
			" var cached = peg$cache[bucket][key];")
			.replace(/peg\$cache\[key\] =/g, "peg$cache[bucket][key] =");

			// Keep these constant throughout an entire tokenization pass.
			tokenizerSource = tokenizerSource
			.replace(/(peg\$FAILED =) {}/, "$1 options.peg$FAILED \|\| {}")
			.replace(/(peg\$cache =) {}/, "$1 options.peg$cache \|\| {}");

			if (!PARSOID_DUMP_TOKENIZER) {
			// eval is not evil in the case of a grammar-generated tokenizer.
			/* jshint evil:true */
			PegTokenizer.prototype.tokenizer = new Function( 'return ' + tokenizerSource )();
			} else {
			// Optionally save & require the tokenizer source
			tokenizerSource = tokenizerSource
			.replace(/peg\$subclass\(child, parent\) {/g, function(m) {
			return m + "\n /* jshint validthis:true */" +
			"\n /* jscs:disable requireCapitalizedConstructors */";
			})
			.replace(/new ctor\(\);/, function(m) {
			return m + "\n /* jscs:enable requireCapitalizedConstructors */";
			});
			tokenizerSource =
			'/* jshint loopfunc:true, latedef:false, nonstandard:true, -W100 */\n' +
			'/* jscs:disable disallowMultipleVarDecl, validateIndentation, requireCamelCaseOrUpperCaseIdentifiers */\n' +
			'\'use strict\';\n' +
			'require(\'./core-upgrade.js\');\n' +
			'module.exports = ' + tokenizerSource + ';';
			// write tokenizer to a file.
			var tokenizerFilename = __dirname + '/mediawiki.tokenizer.js';
			fs.writeFileSync(tokenizerFilename, tokenizerSource, 'utf8');
			PegTokenizer.prototype.tokenizer = require(tokenizerFilename);
			}

			// alias the parse method
			this.tokenizer.tokenize = this.tokenizer.parse;

			};

			/*
			* Process text. The text is tokenized in chunks and control
			* is yielded to the event loop after each top-level block is
			* tokenized enabling the tokenized chunks to be processed at
			* the earliest possible opportunity.
			*/
			PegTokenizer.prototype.process = function( text ) {
			this._processText(text, false);
			};

			/**
			* Debugging aid: set pipeline id
			*/
			PegTokenizer.prototype.setPipelineId = function(id) {
			this.pipelineId = id;
			};

			/**
			* Set start and end offsets of the source that generated this DOM
			*/
			PegTokenizer.prototype.setSourceOffsets = function(start, end) {
			this.offsets.startOffset = start;
			this.offsets.endOffset = end;
			};

			/*
			* Process text synchronously -- the text is tokenized in one shot
			*/
			PegTokenizer.prototype.processSync = function( text ) {
			this._processText(text, true);
			};

			/*
			* The main worker. Sets up event emission ('chunk' and 'end' events).
			* Consumers are supposed to register with PegTokenizer before calling
			* process().
			*/
			PegTokenizer.prototype._processText = function( text, fullParse ) {
			if ( !this.tokenizer ) {
			this.initTokenizer();
			}

			// ensure we're processing text
			text = String(text \|\| "");

			// Some input normalization: force a trailing newline
			// if ( text.substring(text.length - 1) !== "\n" ) {
			// text += "\n";
			// }

			var chunkCB = this.emit.bind( this, 'chunk' );

			// Kick it off!
			var pipelineOffset = this.offsets.startOffset \|\| 0;
			var args = {
			cb: chunkCB,
			pegTokenizer: this,
			pipelineOffset: pipelineOffset,
			env: this.env,
			pegIncludes: pegIncludes,
			startRule: "start"
			};
			if (fullParse) {
			try {
			this.tokenizer.tokenize( text, args );
			} catch (e) {
			this.env.log("fatal", e);
			return;
			}
			this.onEnd();
			} else {
			this.tokenizeAsync(text, 0, 0, pipelineOffset, null, chunkCB);
			}
			};

			PegTokenizer.prototype.tokenizeAsync = function( text, prevOffset, startOffset, pipelineOffset, peg, cb ) {
			// Initialize a some peg vars scoped to this chain
			if ( !peg ) {
			peg = { cache: {}, FAILED: {} };
			}

			var ret;
			var pegTokenizer = this;
			var args = {
			cb: cb,
			pegTokenizer: this,
			pipelineOffset: pipelineOffset,
			startOffset: startOffset,
			prevOffset: prevOffset,
			env: this.env,
			pegIncludes: pegIncludes,
			startRule: "toplevelblock",
			peg$cache: peg.cache,
			peg$FAILED: peg.FAILED,
			};

			try {
			ret = this.tokenizer.tokenize( text, args );
			} catch (e) {
			this.env.log("fatal", e);
			return;
			}

			if (ret.eof) {
			this.onEnd();
			} else {
			// Schedule parse of next chunk
			setImmediate(function() {
			pegTokenizer.tokenizeAsync( text, startOffset, ret.newOffset, pipelineOffset, peg, cb );
			});
			}
			};


			PegTokenizer.prototype.onEnd = function( ) {
			// Reset source offsets
			this.offsets.startOffset = undefined;
			this.offsets.endOffset = undefined;
			this.emit('end');
			};

			/**
			* Tokenize via a rule passed in as an arg.
			* The text is tokenized synchronously in one shot.
			*/
			PegTokenizer.prototype.tokenize = function(text, rule, args, throwErr) {
			if ( !this.tokenizer ) {
			this.initTokenizer();
			}

			try {
			// Some rules use callbacks: start, tlb, toplevelblock.
			// All other rules return tokens directly.
			var toks = [];
			if (!args) {
			args = {
			cb: function(r) { toks = JSUtils.pushArray(toks, r); },
			pegTokenizer: this,
			pipelineOffset: this.offsets.startOffset \|\| 0,
			env: this.env,
			pegIncludes: pegIncludes,
			startRule: rule \|\| "start"
			};
			}
			var retToks = this.tokenizer.tokenize( text, args );

			if ( Array.isArray(retToks) && retToks.length > 0 ) {
			toks = JSUtils.pushArray(toks, retToks);
			}
			return toks;
			} catch ( e ) {
			if (throwErr) {
			throw e; // don't suppress errors
			} else {
			// console.warn("Input: " + text);
			// console.warn("Rule : " + rule);
			// console.warn("ERROR: " + e);
			// console.warn("Stack: " + e.stack);
			return false;
			}
			}
			};

			/**
			* Tokenize a URL
			*/
			PegTokenizer.prototype.tokenizeURL = function( text ) {
			var args = {
			pegTokenizer: this,
			env: this.env,
			pegIncludes: pegIncludes,
			startRule: "url"
			};
			return this.tokenize( text, null, args );
			};

			/**
			* Tokenize table cell attributes
			*/
			PegTokenizer.prototype.tokenizeTableCellAttributes = function( text ) {
			var args = {
			pegTokenizer: this,
			env: this.env,
			pegIncludes: pegIncludes,
			startRule: "single_cell_table_args"
			};
			return this.tokenize( text, null, args );
			};

			if (require.main === module) {
			PARSOID_DUMP_TOKENIZER = true;
			new PegTokenizer().initTokenizer();
			} else if (typeof module === "object") {
			module.exports.PegTokenizer = PegTokenizer;
			}