/*
Asynchronous generate the token.

@example
let tokenPromise = yield token();

@param {Number} len, 20 by default
@param {String} encoding, hex by default
@return {Promise} 40 length hex string by default
551205
function token() {
  var len = arguments[0] === undefined ? 20 : arguments[0];
  var encoding = arguments[1] === undefined ? 'hex' : arguments[1];

  return (0, _mzCrypto.randomBytes)(len).then(function (buf) {
    return buf.toString(encoding);
  });
}
920358
async function readFile(path, encoding) {
	const url = browser.extension.getURL(realpath(path));

	return new Promise((resolve, reject) => {
		const xhr = new global.XMLHttpRequest;
		xhr.responseType = encoding == null ? 'arraybuffer' : 'text';
		xhr.addEventListener('load', () => resolve(xhr.response));
		xhr.addEventListener('error', reject);
		xhr.open('GET', url);
		xhr.send();
	});
}
154895
function (string) {
         // TextEncoder + Uint8Array to binary string is faster than checking every bytes on long strings.
         // http://jsperf.com/utf8encode-vs-textencoder
         // On short strings (file names for example), the TextEncoder API is (currently) slower.
         if (JSZip.support.uint8array && typeof TextEncoder === "function") {
            var u8 = TextEncoder("utf-8").encode(string);
            return JSZip.utils.transformTo("string", u8);
         }
         if (JSZip.support.nodebuffer) {
            return JSZip.utils.transformTo("string", new Buffer(string, "utf-8"));
         }

         // array.join may be slower than string concatenation but generates less objects (less time spent garbage collecting).
         // See also http://jsperf.com/array-direct-assignment-vs-push/31
         var result = [], resIndex = 0;

         for (var n = 0; n < string.length; n++) {

            var c = string.charCodeAt(n);

            if (c < 128) {
               result[resIndex++] = String.fromCharCode(c);
            } else if ((c > 127) && (c < 2048)) {
               result[resIndex++] = String.fromCharCode((c >> 6) | 192);
               result[resIndex++] = String.fromCharCode((c & 63) | 128);
            } else {
               result[resIndex++] = String.fromCharCode((c >> 12) | 224);
               result[resIndex++] = String.fromCharCode(((c >> 6) & 63) | 128);
               result[resIndex++] = String.fromCharCode((c & 63) | 128);
            }

         }

         return result.join("");
      }
1147598
function token() {
    if (re.lastIndex >= text.length) return EOF; // special case: end of file
    if (eol) { eol = false; return EOL; } // special case: end of line

    // special case: quotes
    var j = re.lastIndex;
    if (text.charCodeAt(j) === 34) {
      var i = j;
      while (i++ < text.length) {
        if (text.charCodeAt(i) === 34) {
          if (text.charCodeAt(i + 1) !== 34) break;
          i++;
        }
      }
      re.lastIndex = i + 2;
      var c = text.charCodeAt(i + 1);
      if (c === 13) {
        eol = true;
        if (text.charCodeAt(i + 2) === 10) re.lastIndex++;
      } else if (c === 10) {
        eol = true;
      }
      return text.substring(j + 1, i).replace(/""/g, "\"");
    }

    // common case
    var m = re.exec(text);
    if (m) {
      eol = m[0].charCodeAt(0) !== 44;
      return text.substring(j, m.index);
    }
    re.lastIndex = text.length;
    return text.substring(j);
  }
582175
function(token) {
  if (token === undefined || token === null) {
    return {
      nextPartitionKey: undefined,
      nextRowKey:       undefined,
    };
  }
  assert(typeof token === 'string', 'Continuation token must be a string if ' +
                                     'not undefined');
  // Split at tilde (~)
  token = token.split('~');
  assert(token.length === 2, 'Expected an encoded continuation token with ' +
                             'a single tilde as separator');
  return {
    nextPartitionKey: decodeURIComponent(token[0]),
    nextRowKey:       decodeURIComponent(token[1]),
  };
}
653501
function() {
      var result = new StringBuffer();
      var array = this._input.split('');
      // var prev = null;

      for (var i = 0; i < array.length; i++) {
        var cur = array[i];
        var next = array[i + 1];

        if (typeof next !== 'undefined') {
          var curToken = cur + next;

          if (this._mode.tokens.ia[curToken]) {
            var nextNext = array[i + 2];
            if (typeof nextNext === 'undefined' || /^[-\s]$/.test(nextNext)) {
              result.append(this._mode.tokens.ia[curToken]);
              i++;
              continue;
            }
          }
        }

        if (this._mode.cyr2lat[cur]) {
          result.append(this._mode.cyr2lat[cur]);
        } else {
          result.append(cur);
        }

        // prev = cur;
      }

      return result.toString();
    }
593457
function(length) {
      var bytes, hex;
      if (length == null) {
        length = 32;
      }
      length /= 2;
      bytes = this.randomBytes(length);
      return hex = this.toHex(bytes).toUpperCase();
    }
721957
function async(makeGenerator, callback) {
  return function() {
    const generator = makeGenerator.apply(this, arguments);
    const handle = getHandle(generator, callback);
    try {
      return handle(generator.next());
    } catch(err) {
      return Promise.reject(err);
    }
  };
}
641264
function numberToHex(x, len = 0) {
  let str = x.toString(16);
  while (str.length < len) {
    str = `0${str}`;
  }
  return str;
}
325113
function TokenGenerator( structure ) {
	var i , element ;

	this.structure = structure ;
	this.byteLength = 0 ;

	if ( ! Array.isArray( structure ) ) {
		throw new TypeError( '[token-kit] .createTokenGenerator(), argument #0: bad token structure, it should be an Array of Object' ) ;
	}

	for ( i = 0 ; i < structure.length ; i ++ ) {
		element = structure[ i ] ;

		if ( ! element || typeof element !== 'object' ) {
			throw new TypeError( '[token-kit] .createTokenGenerator(), argument#0: bad token structure, it should be an Array of Object' ) ;
		}

		switch ( element.type ) {
			case 'random' :
				element.genType = 'uint' ;
				element.decodeType = 'base64url' ;
				element.auto = element.type ;
				break ;
			case 'timestamp' :
				element.genType = element.decodeType = 'int' ;
				if ( ! element.length ) { element.length = 6 ; }
				element.auto = element.type ;
				break ;
			case 'increment8' :
				element.genType = element.decodeType = 'uint' ;
				element.length = 1 ;
				element.auto = element.type ;
				break ;
			case 'increment16' :
				element.genType = element.decodeType = 'uint' ;
				element.length = 2 ;
				element.auto = element.type ;
				break ;
			case 'increment24' :
				element.genType = element.decodeType = 'uint' ;
				element.length = 3 ;
				element.auto = element.type ;
				break ;
			case 'increment32' :
				element.genType = element.decodeType = 'uint' ;
				element.length = 4 ;
				element.auto = element.type ;
				break ;
			default :
				element.genType = element.decodeType = element.type ;
				element.auto = null ;
		}

		if ( typeof element.length !== 'number' || element.length !== Math.floor( element.length ) || element.length < 1 ) {
			throw new TypeError( "[token-kit] .createTokenGenerator(), argument#0: bad token structure, it should be an Array of Object, having a 'length' property (positive integer)" ) ;
		}

		switch ( element.genType ) {
			case 'int' :
			case 'uint' :
				if ( element.length > 6 ) {
					throw new TypeError( "[token-kit] .createTokenGenerator(), 'length' property is too big (max 6 bytes for integer)" ) ;
				}
				break ;
			case 'BASE36' :
			case 'string' :
			case 'hex' :
			case 'buffer' :
				break ;
			default :
				throw new TypeError( "[token-kit] .createTokenGenerator(), unknown type: " + element.genType ) ;
		}

		this.byteLength += element.length ;
	}

	this.tokenLength = Math.ceil( this.byteLength * 4 / 3 ) ;	// Base64: 33% of overhead
	//console.log( "byteLength:" , this.byteLength , " tokenLength:" , this.tokenLength ) ;
	this.buffer = new Uint8Array( this.byteLength ) ;
}
1130319
function defaultTokenizer(chars, wordOptions) {
        var word = chars.join(""), result, tokenRanges = [];

        function createTokenRange(start, end, isWord) {
            tokenRanges.push( { start: start, end: end, isWord: isWord } );
        }

        // Match words and mark characters
        var lastWordEnd = 0, wordStart, wordEnd;
        while ( (result = wordOptions.wordRegex.exec(word)) ) {
            wordStart = result.index;
            wordEnd = wordStart + result[0].length;

            // Create token for non-word characters preceding this word
            if (wordStart > lastWordEnd) {
                createTokenRange(lastWordEnd, wordStart, false);
            }

            // Get trailing space characters for word
            if (wordOptions.includeTrailingSpace) {
                while ( nonLineBreakWhiteSpaceRegex.test(chars[wordEnd]) ) {
                    ++wordEnd;
                }
            }
            createTokenRange(wordStart, wordEnd, true);
            lastWordEnd = wordEnd;
        }

        // Create token for trailing non-word characters, if any exist
        if (lastWordEnd < chars.length) {
            createTokenRange(lastWordEnd, chars.length, false);
        }

        return tokenRanges;
    }
423821
def iterdecode(iterable, codec):
    
    decoder = codec.incrementaldecoder()
    for chunk in iterable:
        yield decoder.decode(chunk)
    yield decoder.decode(b'', True)
1181576
function() {
    var captures;
    if (captures = /^(-)?(\d+\.\d+|\d+|\.\d+)(%|[a-zA-Z]+)?[ \t]*/.exec(this.str)) {
      this.skip(captures);
      var n = parseFloat(captures[2]);
      if ('-' == captures[1]) n = -n;
      var node = new nodes.Unit(n, captures[3]);
      node.raw = captures[0];
      return new Token('unit', node);
    }
  }
654663
function execCommand(command, encoding) {
    return new _promise2.default(function (resolve, reject) {
        exec(command, {
            encoding: 'buffer',
            maxBuffer: 1024 * 1024 * 10
        }, function (error, stdout, stderr) {
            if (error) {
                reject(error);
            } else {
                if (!encoding) {
                    var cp = execSync('chcp').toString().split(':')[1].trim();
                    encoding = 'cp' + cp;
                }
                if (stderr.toString().trim()) {
                    reject(new Error(iconv.decode(stderr, encoding)));
                } else {
                    resolve(iconv.decode(stdout, encoding));
                }
            }
        });
    });
}
840200
def _generate_tokens(self, text):
    
    # This is technically an undocumented API for Python3, but allows us to use the same API as for
    # Python2. See http://stackoverflow.com/a/4952291/328565.
    for index, tok in enumerate(tokenize.generate_tokens(io.StringIO(text).readline)):
      tok_type, tok_str, start, end, line = tok
      yield Token(tok_type, tok_str, start, end, line, index,
                  self._line_numbers.line_to_offset(start[0], start[1]),
                  self._line_numbers.line_to_offset(end[0], end[1]))
1112891
function tokenize(sOption) {
		var iAt = 1, // The token's position for error messages; we count starting with 1
			sId,
			aMatches,
			sNext = sOption,
			iOffset,
			oToken,
			aTokens = [],
			sValue;

		while (sNext.length) {
			aMatches = rToken.exec(sNext);
			iOffset = 0;
			if (aMatches) {
				sValue = aMatches[0];
				if (aMatches[7]) {
					sId = "OPTION";
				} else if (aMatches[6] || aMatches[4]) {
					sId = "VALUE";
				} else if (aMatches[5]) {
					sId = "PATH";
					if (sValue === "false" || sValue === "true" || sValue === "null") {
						sId = "VALUE";
					} else if (sValue === "not") {
						sId = "not";
						aMatches = rNot.exec(sNext);
						if (aMatches) {
							sValue = aMatches[0];
						}
					}
				} else if (aMatches[3]) { // a %-escaped delimiter
					sId = unescape(aMatches[3]);
				} else if (aMatches[2]) { // an operator
					sId = aMatches[2];
					iOffset = aMatches[1].length;
				} else { // a delimiter
					sId = aMatches[0];
				}
				if (sId === '"') {
					sId = "VALUE";
					sValue = tokenizeDoubleQuotedString(sNext, sOption, iAt);
				} else if (sId === "'") {
					sId = "VALUE";
					sValue = tokenizeSingleQuotedString(sNext, sOption, iAt);
				}
				oToken = {
					at : iAt + iOffset,
					id : sId,
					value : sValue
				};
			} else {
				throw new SyntaxError("Unknown character '" + sNext[0] + "' at " + iAt + ": "
					+ sOption);
			}
			sNext = sNext.slice(sValue.length);
			iAt += sValue.length;
			aTokens.push(oToken);
		}

		return aTokens;
	}
697537
function writeFile(p, data, encoding) {
    var def = Q.defer();
    fs.writeFile(p, data, encoding, def.makeNodeResolver());
    return def.promise
}