/usr/share/php/Nette/Utils/Tokenizer.php is in php-nette 2.1.0-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | <?php
/**
* This file is part of the Nette Framework (http://nette.org)
* Copyright (c) 2004 David Grudl (http://davidgrudl.com)
*/
namespace Nette\Utils;
use Nette;
/**
* Simple lexical analyser. Internal class.
*
* @author David Grudl
* @internal
*/
class Tokenizer extends Nette\Object
{
const VALUE = 0,
OFFSET = 1,
TYPE = 2;
/** @var string */
private $re;
/** @var array */
private $types;
/**
* @param array of [(int) symbol type => pattern]
* @param string regular expression flag
*/
public function __construct(array $patterns, $flags = '')
{
$this->re = '~(' . implode(')|(', $patterns) . ')~A' . $flags;
$keys = array_keys($patterns);
$this->types = $keys === range(0, count($patterns) - 1) ? FALSE : $keys;
}
/**
* Tokenize string.
* @param string
* @return array
*/
public function tokenize($input)
{
if ($this->types) {
$tokens = Strings::matchAll($input, $this->re);
$len = 0;
$count = count($this->types);
foreach ($tokens as & $match) {
$type = NULL;
for ($i = 1; $i <= $count; $i++) {
if (!isset($match[$i])) {
break;
} elseif ($match[$i] != NULL) {
$type = $this->types[$i - 1]; break;
}
}
$match = array(self::VALUE => $match[0], self::OFFSET => $len, self::TYPE => $type);
$len += strlen($match[self::VALUE]);
}
if ($len !== strlen($input)) {
$errorOffset = $len;
}
} else {
$tokens = Strings::split($input, $this->re, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_OFFSET_CAPTURE);
$last = end($tokens);
if ($tokens && !Strings::match($last[0], $this->re)) {
$errorOffset = $last[1];
}
}
if (isset($errorOffset)) {
list($line, $col) = $this->getCoordinates($input, $errorOffset);
$token = str_replace("\n", '\n', substr($input, $errorOffset, 10));
throw new TokenizerException("Unexpected '$token' on line $line, column $col.");
}
return $tokens;
}
/**
* Returns position of token in input string.
* @param int token number
* @return array [line, column]
*/
public static function getCoordinates($text, $offset)
{
$text = substr($text, 0, $offset);
return array(substr_count($text, "\n") + 1, $offset - strrpos("\n" . $text, "\n") + 1);
}
}
/**
* The exception that indicates tokenizer error.
*/
class TokenizerException extends \Exception
{
}
|