Просмотр файла engine/classes/lib/Twig/Lexer.php

Конкурс дизайнеров на 15000 рублей
Сервис рекламы
Размер файла: 15.79Kb 
  1. <?php
  2.  
  3. /*
  4.  * This file is part of Twig.
  5.  *
  6.  * (c) 2009 Fabien Potencier
  7.  * (c) 2009 Armin Ronacher
  8.  *
  9.  * For the full copyright and license information, please view the LICENSE
  10.  * file that was distributed with this source code.
  11.  */
  12.  
  13. /**
  14.  * Lexes a template string.
  15.  *
  16.  * @author Fabien Potencier <fabien@symfony.com>
  17.  */
  18. class Twig_Lexer implements Twig_LexerInterface
  19. {
  20.     protected $tokens;
  21.     protected $code;
  22.     protected $cursor;
  23.     protected $lineno;
  24.     protected $end;
  25.     protected $state;
  26.     protected $states;
  27.     protected $brackets;
  28.     protected $env;
  29.     protected $filename;
  30.     protected $options;
  31.     protected $regexes;
  32.     protected $position;
  33.     protected $positions;
  34.     protected $currentVarBlockLine;
  35.  
  36.     const STATE_DATA            = 0;
  37.     const STATE_BLOCK           = 1;
  38.     const STATE_VAR             = 2;
  39.     const STATE_STRING          = 3;
  40.     const STATE_INTERPOLATION   = 4;
  41.  
  42.     const REGEX_NAME            = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A';
  43.     const REGEX_NUMBER          = '/[0-9]+(?:\.[0-9]+)?/A';
  44.     const REGEX_STRING          = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
  45.     const REGEX_DQ_STRING_DELIM = '/"/A';
  46.     const REGEX_DQ_STRING_PART  = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As';
  47.     const PUNCTUATION           = '()[]{}?:.,|';
  48.  
  49.     public function __construct(Twig_Environment $env, array $options = array())
  50.     {
  51.         $this->env = $env;
  52.  
  53.         $this->options = array_merge(array(
  54.             'tag_comment'     => array('{#', '#}'),
  55.             'tag_block'       => array('{%', '%}'),
  56.             'tag_variable'    => array('{{', '}}'),
  57.             'whitespace_trim' => '-',
  58.             'interpolation'   => array('#{', '}'),
  59.         ), $options);
  60.  
  61.         $this->regexes = array(
  62.             'lex_var'             => '/\s*'.preg_quote($this->options['whitespace_trim'].$this->options['tag_variable'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_variable'][1], '/').'/A',
  63.             'lex_block'           => '/\s*(?:'.preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_block'][1], '/').')\n?/A',
  64.             'lex_raw_data'        => '/('.preg_quote($this->options['tag_block'][0].$this->options['whitespace_trim'], '/').'|'.preg_quote($this->options['tag_block'][0], '/').')\s*(?:end%s)\s*(?:'.preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_block'][1], '/').')/s',
  65.             'operator'            => $this->getOperatorRegex(),
  66.             'lex_comment'         => '/(?:'.preg_quote($this->options['whitespace_trim'], '/').preg_quote($this->options['tag_comment'][1], '/').'\s*|'.preg_quote($this->options['tag_comment'][1], '/').')\n?/s',
  67.             'lex_block_raw'       => '/\s*(raw|verbatim)\s*(?:'.preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_block'][1], '/').')/As',
  68.             'lex_block_line'      => '/\s*line\s+(\d+)\s*'.preg_quote($this->options['tag_block'][1], '/').'/As',
  69.             'lex_tokens_start'    => '/('.preg_quote($this->options['tag_variable'][0], '/').'|'.preg_quote($this->options['tag_block'][0], '/').'|'.preg_quote($this->options['tag_comment'][0], '/').')('.preg_quote($this->options['whitespace_trim'], '/').')?/s',
  70.             'interpolation_start' => '/'.preg_quote($this->options['interpolation'][0], '/').'\s*/A',
  71.             'interpolation_end'   => '/\s*'.preg_quote($this->options['interpolation'][1], '/').'/A',
  72.         );
  73.     }
  74.  
  75.     /**
  76.      * {@inheritdoc}
  77.      */
  78.     public function tokenize($code, $filename = null)
  79.     {
  80.         if (function_exists('mb_internal_encoding') && ((int) ini_get('mbstring.func_overload')) & 2) {
  81.             $mbEncoding = mb_internal_encoding();
  82.             mb_internal_encoding('ASCII');
  83.         }
  84.  
  85.         $this->code = str_replace(array("\r\n", "\r"), "\n", $code);
  86.         $this->filename = $filename;
  87.         $this->cursor = 0;
  88.         $this->lineno = 1;
  89.         $this->end = strlen($this->code);
  90.         $this->tokens = array();
  91.         $this->state = self::STATE_DATA;
  92.         $this->states = array();
  93.         $this->brackets = array();
  94.         $this->position = -1;
  95.  
  96.         // find all token starts in one go
  97.         preg_match_all($this->regexes['lex_tokens_start'], $this->code, $matches, PREG_OFFSET_CAPTURE);
  98.         $this->positions = $matches;
  99.  
  100.         while ($this->cursor < $this->end) {
  101.             // dispatch to the lexing functions depending
  102.             // on the current state
  103.             switch ($this->state) {
  104.                 case self::STATE_DATA:
  105.                     $this->lexData();
  106.                     break;
  107.  
  108.                 case self::STATE_BLOCK:
  109.                     $this->lexBlock();
  110.                     break;
  111.  
  112.                 case self::STATE_VAR:
  113.                     $this->lexVar();
  114.                     break;
  115.  
  116.                 case self::STATE_STRING:
  117.                     $this->lexString();
  118.                     break;
  119.  
  120.                 case self::STATE_INTERPOLATION:
  121.                     $this->lexInterpolation();
  122.                     break;
  123.             }
  124.         }
  125.  
  126.         $this->pushToken(Twig_Token::EOF_TYPE);
  127.  
  128.         if (!empty($this->brackets)) {
  129.             list($expect, $lineno) = array_pop($this->brackets);
  130.             throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
  131.         }
  132.  
  133.         if (isset($mbEncoding)) {
  134.             mb_internal_encoding($mbEncoding);
  135.         }
  136.  
  137.         return new Twig_TokenStream($this->tokens, $this->filename);
  138.     }
  139.  
  140.     protected function lexData()
  141.     {
  142.         // if no matches are left we return the rest of the template as simple text token
  143.         if ($this->position == count($this->positions[0]) - 1) {
  144.             $this->pushToken(Twig_Token::TEXT_TYPE, substr($this->code, $this->cursor));
  145.             $this->cursor = $this->end;
  146.  
  147.             return;
  148.         }
  149.  
  150.         // Find the first token after the current cursor
  151.         $position = $this->positions[0][++$this->position];
  152.         while ($position[1] < $this->cursor) {
  153.             if ($this->position == count($this->positions[0]) - 1) {
  154.                 return;
  155.             }
  156.             $position = $this->positions[0][++$this->position];
  157.         }
  158.  
  159.         // push the template text first
  160.         $text = $textContent = substr($this->code, $this->cursor, $position[1] - $this->cursor);
  161.         if (isset($this->positions[2][$this->position][0])) {
  162.             $text = rtrim($text);
  163.         }
  164.         $this->pushToken(Twig_Token::TEXT_TYPE, $text);
  165.         $this->moveCursor($textContent.$position[0]);
  166.  
  167.         switch ($this->positions[1][$this->position][0]) {
  168.             case $this->options['tag_comment'][0]:
  169.                 $this->lexComment();
  170.                 break;
  171.  
  172.             case $this->options['tag_block'][0]:
  173.                 // raw data?
  174.                 if (preg_match($this->regexes['lex_block_raw'], $this->code, $match, null, $this->cursor)) {
  175.                     $this->moveCursor($match[0]);
  176.                     $this->lexRawData($match[1]);
  177.                 // {% line \d+ %}
  178.                 } elseif (preg_match($this->regexes['lex_block_line'], $this->code, $match, null, $this->cursor)) {
  179.                     $this->moveCursor($match[0]);
  180.                     $this->lineno = (int) $match[1];
  181.                 } else {
  182.                     $this->pushToken(Twig_Token::BLOCK_START_TYPE);
  183.                     $this->pushState(self::STATE_BLOCK);
  184.                     $this->currentVarBlockLine = $this->lineno;
  185.                 }
  186.                 break;
  187.  
  188.             case $this->options['tag_variable'][0]:
  189.                 $this->pushToken(Twig_Token::VAR_START_TYPE);
  190.                 $this->pushState(self::STATE_VAR);
  191.                 $this->currentVarBlockLine = $this->lineno;
  192.                 break;
  193.         }
  194.     }
  195.  
  196.     protected function lexBlock()
  197.     {
  198.         if (empty($this->brackets) && preg_match($this->regexes['lex_block'], $this->code, $match, null, $this->cursor)) {
  199.             $this->pushToken(Twig_Token::BLOCK_END_TYPE);
  200.             $this->moveCursor($match[0]);
  201.             $this->popState();
  202.         } else {
  203.             $this->lexExpression();
  204.         }
  205.     }
  206.  
  207.     protected function lexVar()
  208.     {
  209.         if (empty($this->brackets) && preg_match($this->regexes['lex_var'], $this->code, $match, null, $this->cursor)) {
  210.             $this->pushToken(Twig_Token::VAR_END_TYPE);
  211.             $this->moveCursor($match[0]);
  212.             $this->popState();
  213.         } else {
  214.             $this->lexExpression();
  215.         }
  216.     }
  217.  
  218.     protected function lexExpression()
  219.     {
  220.         // whitespace
  221.         if (preg_match('/\s+/A', $this->code, $match, null, $this->cursor)) {
  222.             $this->moveCursor($match[0]);
  223.  
  224.             if ($this->cursor >= $this->end) {
  225.                 throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $this->state === self::STATE_BLOCK ? 'block' : 'variable'), $this->currentVarBlockLine, $this->filename);
  226.             }
  227.         }
  228.  
  229.         // operators
  230.         if (preg_match($this->regexes['operator'], $this->code, $match, null, $this->cursor)) {
  231.             $this->pushToken(Twig_Token::OPERATOR_TYPE, preg_replace('/\s+/', ' ', $match[0]));
  232.             $this->moveCursor($match[0]);
  233.         }
  234.         // names
  235.         elseif (preg_match(self::REGEX_NAME, $this->code, $match, null, $this->cursor)) {
  236.             $this->pushToken(Twig_Token::NAME_TYPE, $match[0]);
  237.             $this->moveCursor($match[0]);
  238.         }
  239.         // numbers
  240.         elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, null, $this->cursor)) {
  241.             $number = (float) $match[0];  // floats
  242.             if (ctype_digit($match[0]) && $number <= PHP_INT_MAX) {
  243.                 $number = (int) $match[0]; // integers lower than the maximum
  244.             }
  245.             $this->pushToken(Twig_Token::NUMBER_TYPE, $number);
  246.             $this->moveCursor($match[0]);
  247.         }
  248.         // punctuation
  249.         elseif (false !== strpos(self::PUNCTUATION, $this->code[$this->cursor])) {
  250.             // opening bracket
  251.             if (false !== strpos('([{', $this->code[$this->cursor])) {
  252.                 $this->brackets[] = array($this->code[$this->cursor], $this->lineno);
  253.             }
  254.             // closing bracket
  255.             elseif (false !== strpos(')]}', $this->code[$this->cursor])) {
  256.                 if (empty($this->brackets)) {
  257.                     throw new Twig_Error_Syntax(sprintf('Unexpected "%s"', $this->code[$this->cursor]), $this->lineno, $this->filename);
  258.                 }
  259.  
  260.                 list($expect, $lineno) = array_pop($this->brackets);
  261.                 if ($this->code[$this->cursor] != strtr($expect, '([{', ')]}')) {
  262.                     throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
  263.                 }
  264.             }
  265.  
  266.             $this->pushToken(Twig_Token::PUNCTUATION_TYPE, $this->code[$this->cursor]);
  267.             ++$this->cursor;
  268.         }
  269.         // strings
  270.         elseif (preg_match(self::REGEX_STRING, $this->code, $match, null, $this->cursor)) {
  271.             $this->pushToken(Twig_Token::STRING_TYPE, stripcslashes(substr($match[0], 1, -1)));
  272.             $this->moveCursor($match[0]);
  273.         }
  274.         // opening double quoted string
  275.         elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, null, $this->cursor)) {
  276.             $this->brackets[] = array('"', $this->lineno);
  277.             $this->pushState(self::STATE_STRING);
  278.             $this->moveCursor($match[0]);
  279.         }
  280.         // unlexable
  281.         else {
  282.             throw new Twig_Error_Syntax(sprintf('Unexpected character "%s"', $this->code[$this->cursor]), $this->lineno, $this->filename);
  283.         }
  284.     }
  285.  
  286.     protected function lexRawData($tag)
  287.     {
  288.         if (!preg_match(str_replace('%s', $tag, $this->regexes['lex_raw_data']), $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor)) {
  289.             throw new Twig_Error_Syntax(sprintf('Unexpected end of file: Unclosed "%s" block', $tag), $this->lineno, $this->filename);
  290.         }
  291.  
  292.         $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor);
  293.         $this->moveCursor($text.$match[0][0]);
  294.  
  295.         if (false !== strpos($match[1][0], $this->options['whitespace_trim'])) {
  296.             $text = rtrim($text);
  297.         }
  298.  
  299.         $this->pushToken(Twig_Token::TEXT_TYPE, $text);
  300.     }
  301.  
  302.     protected function lexComment()
  303.     {
  304.         if (!preg_match($this->regexes['lex_comment'], $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor)) {
  305.             throw new Twig_Error_Syntax('Unclosed comment', $this->lineno, $this->filename);
  306.         }
  307.  
  308.         $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
  309.     }
  310.  
  311.     protected function lexString()
  312.     {
  313.         if (preg_match($this->regexes['interpolation_start'], $this->code, $match, null, $this->cursor)) {
  314.             $this->brackets[] = array($this->options['interpolation'][0], $this->lineno);
  315.             $this->pushToken(Twig_Token::INTERPOLATION_START_TYPE);
  316.             $this->moveCursor($match[0]);
  317.             $this->pushState(self::STATE_INTERPOLATION);
  318.  
  319.         } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, null, $this->cursor) && strlen($match[0]) > 0) {
  320.             $this->pushToken(Twig_Token::STRING_TYPE, stripcslashes($match[0]));
  321.             $this->moveCursor($match[0]);
  322.  
  323.         } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, null, $this->cursor)) {
  324.             list($expect, $lineno) = array_pop($this->brackets);
  325.             if ($this->code[$this->cursor] != '"') {
  326.                 throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
  327.             }
  328.  
  329.             $this->popState();
  330.             ++$this->cursor;
  331.         }
  332.     }
  333.  
  334.     protected function lexInterpolation()
  335.     {
  336.         $bracket = end($this->brackets);
  337.         if ($this->options['interpolation'][0] === $bracket[0] && preg_match($this->regexes['interpolation_end'], $this->code, $match, null, $this->cursor)) {
  338.             array_pop($this->brackets);
  339.             $this->pushToken(Twig_Token::INTERPOLATION_END_TYPE);
  340.             $this->moveCursor($match[0]);
  341.             $this->popState();
  342.         } else {
  343.             $this->lexExpression();
  344.         }
  345.     }
  346.  
  347.     protected function pushToken($type, $value = '')
  348.     {
  349.         // do not push empty text tokens
  350.         if (Twig_Token::TEXT_TYPE === $type && '' === $value) {
  351.             return;
  352.         }
  353.  
  354.         $this->tokens[] = new Twig_Token($type, $value, $this->lineno);
  355.     }
  356.  
  357.     protected function moveCursor($text)
  358.     {
  359.         $this->cursor += strlen($text);
  360.         $this->lineno += substr_count($text, "\n");
  361.     }
  362.  
  363.     protected function getOperatorRegex()
  364.     {
  365.         $operators = array_merge(
  366.             array('='),
  367.             array_keys($this->env->getUnaryOperators()),
  368.             array_keys($this->env->getBinaryOperators())
  369.         );
  370.  
  371.         $operators = array_combine($operators, array_map('strlen', $operators));
  372.         arsort($operators);
  373.  
  374.         $regex = array();
  375.         foreach ($operators as $operator => $length) {
  376.             // an operator that ends with a character must be followed by
  377.             // a whitespace or a parenthesis
  378.             if (ctype_alpha($operator[$length - 1])) {
  379.                 $r = preg_quote($operator, '/').'(?=[\s()])';
  380.             } else {
  381.                 $r = preg_quote($operator, '/');
  382.             }
  383.  
  384.             // an operator with a space can be any amount of whitespaces
  385.             $r = preg_replace('/\s+/', '\s+', $r);
  386.  
  387.             $regex[] = $r;
  388.         }
  389.  
  390.         return '/'.implode('|', $regex).'/A';
  391.     }
  392.  
  393.     protected function pushState($state)
  394.     {
  395.         $this->states[] = $this->state;
  396.         $this->state = $state;
  397.     }
  398.  
  399.     protected function popState()
  400.     {
  401.         if (0 === count($this->states)) {
  402.             throw new Exception('Cannot pop state without a previous state');
  403.         }
  404.  
  405.         $this->state = array_pop($this->states);
  406.     }
  407. }