JezK
Edit File: tokenize.cpython-39.opt-2.pyc
a �iR � @ s� d Z dZddlZddlZddlmZmZ ddlT ddlm Z dd � e e �D �g d � Z [ ze W n e yv eZ Y n0 d d � Zd d� Zdd� Zdd� ZdZdZeede � ee� ZdZdZdZdZedd�Zeeeee�Zd Z ed d �ee � Z d e Z ee e �Z ed e d! �Z ee e e�Z!d"Z"d#Z#d$Z$d%Z%d&Z&ee&d' e&d( �Z'ee&d) e&d* �Z(ed+d,d-d.d/d0d1d2d3� Z)d4Z*ed5d6d7�Z+ee)e*e+�Z,ee!e,e(e�Z-ee- Z.ee&d8 ed9d� e&d: ed;d� �Z/edee'�Z0eee0e!e,e/e� Z1e2ej3e.e1e$e%f�\Z4Z5Z6Z7ed<d=d>d?�ed<d=d@dA�B h dB�B Z8e�3e"�e�3e#�e6e7dC�dDdE� e8D ��dFdE� e8D ��dGdE� e8D ��Z9d'd(hdHdI� e8D �B dJdI� e8D �B Z:d9d;hdKdI� e8D �B dLdI� e8D �B Z;dMZ<G dNdO� dOe=�Z>G dPdQ� dQe=�Z?dRdS� Z@e@fdTdU�ZAdVdW� ZBG dXdY� dY�ZCe�3dZejD�ZEe�3d[ejD�ZFd\d]� ZGd^d_� ZHd`da� ZIdbdc� ZJeKddk�r�ddlLZLeMeLjN�dk�rxeAeOeLjNd �jP� n eAeLjQjP� dS )ezKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s g | ]}|d dkr|�qS )r �_� )�.0�xr r �./usr/lib64/python3.9/lib2to3/pgen2/tokenize.py� <listcomp>% � r )�tokenize�generate_tokens� untokenizec G s dd� | � d S )N�(�|�))�join��choicesr r r �group0 r r c G s t | � d S )Nr �r r r r r �any1 r r c G s t | � d S )N�?r r r r r �maybe2 r r c s t � fdd�� D ��S )Nc 3 s4 | ],}� d D ] }|� � |� � kr|| V qqdS ))� N)�casefold)r r �y��lr r � <genexpr>4 s z _combinations.<locals>.<genexpr>)�setr r r r � _combinations3 s �r# z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?�'''�"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"�r�R�f�F�b�B> ZUrZUR�ur�uZuR�U)r' r( r$ r% c C s i | ]}|� d �t �qS �r$ )� single3prog�r �prefixr r r � <dictcomp>y r r6 c C s i | ]}|� d �t �qS �r% )� double3progr4 r r r r6 z r c C s i | ] }|d �qS �Nr r4 r r r r6 { r c C s h | ]}|� d ��qS r2 r r4 r r r � <setcomp> r r: c C s h | ]}|� d ��qS r7 r r4 r r r r: � r c C s h | ]}|� d ��qS )r' r r4 r r r r: � r c C s h | ]}|� d ��qS )r( r r4 r r r r: � r � c @ s e Zd ZdS )� TokenErrorN��__name__� __module__� __qualname__r r r r r< � r r< c @ s e Zd ZdS )�StopTokenizingNr= r r r r rA � r rA c C s4 |\}}|\}}t d||||t| t|�f � d S )Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r � printtoken� s �rG c C s& zt | |� W n ty Y n0 d S r9 )� tokenize_looprA )�readline� tokeneaterr r r r � s r c C s t | �D ] }||� qd S r9 )r )rI rJ Z token_infor r r rH � s rH c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) � Untokenizerc C s g | _ d| _d| _d S )Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s* |\}}|| j }|r&| j�d| � d S )N� )rN rL �append)rO �start�row�col� col_offsetr r r �add_whitespace� s zUntokenizer.add_whitespacec C s� |D ]p}t |�dkr$| �||� qv|\}}}}}| �|� | j�|� |\| _| _|ttfv r| jd7 _d| _qd� | j�S )N� r r r ) �len�compatrW rL rR rM rN �NEWLINE�NLr )rO �iterable�t�tok_typer rS �endrF r r r r � s zUntokenizer.untokenizec C s� d}g }| j j}|\}}|ttfv r,|d7 }|ttfv r<d}|D ]�}|d d� \}}|ttttfv rl|d7 }|tkr�|�|� q@n>|t kr�|� � q@n*|ttfv r�d}n|r�|r�||d � d}||� q@d S )NFrQ TrX ���) rL rR �NAME�NUMBERr[ r\ �ASYNC�AWAIT�INDENT�DEDENT�pop) rO r r] � startline�indents� toks_append�toknum�tokval�tokr r r rZ � s0 zUntokenizer.compatN)r> r? r@ rP rW r rZ r r r r rK � s rK z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | d d� � � �dd�}|dks*|�d�r.dS |dv s@|�d�rDdS | S ) N� r �-�utf-8zutf-8-)zlatin-1� iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-rr )�lower�replace� startswith)�orig_enc�encr r r �_get_normal_name� s �rx c s� d� d }d}�fdd�}� fdd�}|� }|� t�rHd� |dd � }d }|sT|g fS ||�}|rj||gfS t�|�s~||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS ) NFrq c s& z� � W S t y t� Y S 0 d S r9 )� StopIteration�bytesr )rI r r � read_or_stop s z%detect_encoding.<locals>.read_or_stopc s� z| � d�}W n ty" Y d S 0 t�|�}|s6d S t|�d��}z t|�}W n tyn td| ��Y n0 � r�|j dkr�td��|d7 }|S )N�asciir zunknown encoding: rq zencoding problem: utf-8z-sig) �decode�UnicodeDecodeError� cookie_re�matchrx r r � LookupError� SyntaxError�name)rF � line_stringr� �encoding�codec)� bom_foundr r � find_cookie s" z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)ru r �blank_rer� )rI r� �defaultr{ r� �first�secondr )r� rI r �detect_encoding� s0 r� c C s t � }|�| �S r9 )rK r )r] �utr r r r : s r c c s| d } }}d\}}d }dg}d }d} d} d} z | � } W n t yN d} Y n0 |d }dt| � } }|�r0| sztd|��|�| �}|r�|�d� } }t|| d |� |||f|| fV d\}}d }nd|�r| dd � dk�r| d d � d k�rt|| ||t| �f|fV d}d }q.n|| }|| }q.�nB|dk�rZ|�sZ| �sJ�q$d}| |k �r�| | d k�rp|d }n8| | d k�r�|t d t }n| | d k�r�d}n�q�| d } �qN| |k�rq$|�r�|V d }| | dv �r�| | dk�rR| | d � �d�}| t|� }t ||| f|| t|� f| fV t | |d � ||f|t| �f| fV q.t t f| | dk | | d � || f|t| �f| fV q.||d k�r�|� |� t | d | � |df|| f| fV ||d k �r2||v�r�t dd|| | f��|d d� }| �r| |d k�rd} d} d} td|| f|| f| fV �q�| �rr| �rr| |d k�rrd} d} d} n| �sntd|df��d}| |k r.t�| | �}|�r�|�d�\}}||f||f| }}} | ||� | | }}|tjv �s�|dk�r�|dk�r�t|||| fV �q |dv �rHt} |dk�rt } n | �r$d} |�r4|V d }| |||| fV �q |dk�rv|�rb|V d }t |||| fV �q |tv �r�t| }|�| | �}|�r�|�d�} | || � }|�r�|V d }t|||| f| fV n||f}| |d � }| }q.�q |tv �s"|d d� tv �s"|d d� tv �r�|d dk�rv||f}t| �pZt|d �pZt|d }| |d � d }}| }q.n |�r�|V d }t|||| fV �q |�� �rz|dv �r�| �r�|dk�r�tnt|||| fV �qrt|||| f} |dk�r�|�s�| }�qr|d v �rb|�rb|d tk�rb|d dk�rb|d k�r>d} |d } t|d |d |d |d fV d }|�rr|V d }| V nz|d k�r�|�r�|V d }t |||| f| fV d}nF|d v �r�|d }n|d!v �r�|d }|�r�|V d }t |||| fV n(t| | || f|| d f| fV | d } �qrq.|�r4|V d }|dd � D ] } td|df|dfdfV �q@t d|df|dfdfV d S )"Nr )r r Fr r zEOF in multi-line string���z\ ���z\ rQ � � z# �#z ra z3unindent does not match any outer indentation levelz <tokenize>zEOF in multi-line statement�.TrX r� � )�async�awaitr� )�def�forr� � �\z([{z)]}) ry rY r< r� r` �STRING� ERRORTOKEN�tabsize�rstrip�COMMENTr\ rR rf �IndentationErrorrg � pseudoprog�span�string�digitsrc r[ � triple_quoted�endprogs� single_quoted� isidentifierrd re rb �OP� ENDMARKER) rI �lnum�parenlev� continued�contstr�needcont�contlinerj ZstashedZ async_defZasync_def_indentZ async_def_nlrF �pos�max�strstart�endprog�endmatchr` �column� comment_tokenZnl_pos� pseudomatchrS �spos�eposr �initial�newlinern �indentr r r r O s� �*� � � � � �� �� � � � � � � r �__main__)R� __author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenr r �dir�__all__rz � NameError�strr r r r# � Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent� Pointfloat�Expfloat� Floatnumber� Imagnumber�Number�Single�Double�Single3�Double3Z _litprefix�Triple�StringZOperatorZBracket�Special�Funny� PlainToken�Token�ContStr� PseudoExtras� PseudoToken�map�compileZ tokenprogr� r3 r8 Z _strprefixesr� r� r� r� � Exceptionr< rA rG r rH rK �ASCIIr r� rx r� r r r> �sysrY �argv�openrI �stdinr r r r �<module> s� � � � ��� � ���� � � � � �� � �� 8 I c &