B
    dj                 @   s@  d dl mZmZmZ d dlmZmZmZmZ G dd de	Z
dd Zdd Zd	d
 Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  ZG d!d" d"e	Zd#d$ Zd%d& ZG d'd( d(e	Zd)d* Zd@d,d-Zd.d/ ZdAd1d2Z d3d4 Z!dBd5d6Z"d7d8 Z#G d9d: d:e	Z$d;d< Z%dCd>d?Z&d+S )D    )absolute_importdivisionprint_function)unsigned_integer_scanidentifier_scan'floating_point_scan_after_exponent_charfloating_point_scan_after_dotc               @   s  e Zd ZddddgZdd Zdd Zd	d
 Zdd Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 ZdGd2d3ZdHd5d6Zd7d8 ZdId9d:Zd;d< Zd=d> Zd?d@ Z dAdB Z!dCdD Z"dEdF Z#d4S )Jtokentypeidssli_codevaluec             C   s   || _ || _|| _|| _d S )N)r
   r   r   r   )Or
   r   r   r    r   q/mnt/filia/a/genomebrowser/www/genomebrowser/fleming/tools/molprobity/modules/cctbx_project/fable/tokenization.py__init__   s    ztoken.__init__c             C   s&   ddddddddd	d
ddddg| j  S )N
identifierintegerZhexadecimalrealZdouble_precisionlogicalstringopcomplexseqZparenthesesZ
implied_dopowerformat)r
   )r   r   r   r   type   s    z
token.typec             C   s
   | j dkS )Nr   )r
   )r   r   r   r   is_identifier#       ztoken.is_identifierc             C   s
   | j dkS )N   )r
   )r   r   r   r   
is_integer$   r   ztoken.is_integerc             C   s
   | j dkS )N   )r
   )r   r   r   r   is_hexadecimal%   r   ztoken.is_hexadecimalc             C   s
   | j dkS )N   )r
   )r   r   r   r   is_real&   r   ztoken.is_realc             C   s
   | j dkS )N   )r
   )r   r   r   r   is_double_precision'   r   ztoken.is_double_precisionc             C   s
   | j dkS )N   )r
   )r   r   r   r   
is_logical(   r   ztoken.is_logicalc             C   s
   | j dkS )N   )r
   )r   r   r   r   	is_string)   r   ztoken.is_stringc             C   s
   | j dkS )N   )r
   )r   r   r   r   is_op*   r   ztoken.is_opc             C   s
   | j dkS )N   )r
   )r   r   r   r   
is_complex+   r   ztoken.is_complexc             C   s
   | j dkS )N	   )r
   )r   r   r   r   is_seq,   r   ztoken.is_seqc             C   s
   | j dkS )N
   )r
   )r   r   r   r   is_parentheses-   r   ztoken.is_parenthesesc             C   s
   | j dkS )N   )r
   )r   r   r   r   is_implied_do.   r   ztoken.is_implied_doc             C   s
   | j dkS )N   )r
   )r   r   r   r   is_power/   r   ztoken.is_powerc             C   s
   | j dkS )N   )r
   )r   r   r   r   	is_format0   r   ztoken.is_formatc             C   s
   | j dkS )Nr%   )r
   )r   r   r   r   is_identifier_or_scalar_number2   s    z$token.is_identifier_or_scalar_numberc             C   s   | j dko| j|kS )Nr+   )r
   r   )r   r   r   r   r   
is_op_with5   s    ztoken.is_op_withc             C   s   | j dko| jdkS )Nr+   )+-)r
   r   )r   r   r   r   is_unary_plus_or_minus8   s    ztoken.is_unary_plus_or_minusc             C   s   d| j   kodkS   S )Nr/   r1   )r
   )r   r   r   r   is_seq_or_parentheses;   s    ztoken.is_seq_or_parenthesesc             C   s   d| j   kodkS   S )Nr/   r3   )r
   )r   r   r   r   #is_seq_or_parentheses_or_implied_do>   s    z)token.is_seq_or_parentheses_or_implied_doc             C   s   | j j| jdS )N)i)r   stmt_locationr   )r   r   r   r   rA   A   s    ztoken.stmt_location c             C   s   | j j|| j|dS )N)msgr@   prefix)r   format_errorr   )r   rC   rD   r   r   r   rE   D   s    ztoken.format_errorNc             C   s   | j j|| j|d d S )N)rC   r@   	ErrorType)r   raise_errorr   )r   rC   rF   r   r   r   rG   G   s    ztoken.raise_errorc             C   s   | j j| jd d S )N)r@   )r   raise_syntax_errorr   )r   r   r   r   rH   J   s    ztoken.raise_syntax_errorc             C   s   | j j|| jd d S )N)rC   r@   )r   raise_semantic_errorr   )r   rC   r   r   r   rI   M   s    ztoken.raise_semantic_errorc             C   s   | j dtd d S )NzSorry: not supported)rC   rF   )rG   RuntimeError)r   r   r   r   raise_not_supportedP   s    ztoken.raise_not_supportedc             C   s   | j j| jd d S )N)r@   )r   raise_internal_errorr   )r   r   r   r   rL   U   s    ztoken.raise_internal_errorc             C   s<   t | jd}|dk s"|t| jkr8| jdt| j d d S )N)coder   zNot an identifier: %s)rC   )r   r   lenrG   repr)r   r@   r   r   r   raise_if_not_identifierX   s    ztoken.raise_if_not_identifierc             C   s   | j dd d S )NzMissing a closing ")")rC   )rG   )r   r   r   r   raise_missing_closing]   s    ztoken.raise_missing_closingc             C   s   | j dd d S )Nz2Closing ")" without a matching opening parenthesis)rC   )rG   )r   r   r   r   raise_missing_opening`   s    ztoken.raise_missing_openingc             C   s   d| j dd d S )N'z'')r   replace)r   r   r   r   fquotec   s    ztoken.fquote)rB   )N)N)$__name__
__module____qualname__	__slots__r   r   r   r    r"   r$   r&   r(   r*   r,   r.   r0   r2   r4   r6   r8   r9   r:   r=   r>   r?   rA   rE   rG   rH   rI   rK   rL   rP   rQ   rR   rU   r   r   r   r   r	      sB   


r	   c             C   s   t d| ||S )Nr   )r	   )r   r   r   r   r   r   tk_identifierf   r   rZ   c             C   s   t d| ||S )Nr   )r	   )r   r   r   r   r   r   
tk_integerg   r   r[   c             C   s   t d| ||S )Nr!   )r	   )r   r   r   r   r   r   tk_hexadecimalh   r   r\   c             C   s   t d| ||S )Nr#   )r	   )r   r   r   r   r   r   tk_reali   r   r]   c             C   s   t d| ||S )Nr%   )r	   )r   r   r   r   r   r   tk_double_precisionj   r   r^   c             C   s   t d| ||S )Nr'   )r	   )r   r   r   r   r   r   
tk_logicalk   r   r_   c             C   s   t d| ||S )Nr)   )r	   )r   r   r   r   r   r   	tk_stringl   r   r`   c             C   s   t d| ||S )Nr+   )r	   )r   r   r   r   r   r   tk_opm   r   ra   c             C   s   t d| ||S )Nr-   )r	   )r   r   r   r   r   r   
tk_complexn   r   rb   c             C   s   t d| ||S )Nr/   )r	   )r   r   r   r   r   r   tk_seqo   r   rc   c             C   s   t d| ||S )Nr1   )r	   )r   r   r   r   r   r   tk_parenthesesp   r   rd   c             C   s   t d| ||S )Nr3   )r	   )r   r   r   r   r   r   tk_implied_doq   r   re   c             C   s   t d| ||S )Nr5   )r	   )r   r   r   r   r   r   tk_powerr   r   rf   c             C   s   t d| ||S )Nr7   )r	   )r   r   r   r   r   r   	tk_formats   r   rg   c               @   s   e Zd ZdZdddddgZd%dd	Zd
d Zdd Zdd Zd&ddZ	dd Z
d'ddZdd Zdd Zd(ddZd)dd Zd!d" Zd#d$ ZdS )*ssl_iteratorzstripped source line iteratorr   startstopr@   bufferNc             C   s8   || _ || _|d kr"t|j| _n|| _|| _g | _d S )N)r   ri   rN   rM   rj   r@   rk   )r   r   ri   rj   r   r   r   r   z   s     zssl_iterator.__init__c             C   s   | S )Nr   )r   r   r   r   __iter__   r   zssl_iterator.__iter__c             C   s   | j dd}|d krt|S )NT)optional)getStopIteration)selftokr   r   r   __next__   s    zssl_iterator.__next__c             C   s   |   S )N)rr   )rp   r   r   r   next   s    zssl_iterator.nextFc       
         s   j j} j fdd}t jdkrV jd\ _}|d krR|  |S  jd krxrhd S jd d xx jk r j}|| }d|dkr|	d|d r܈  jd7  _t
||||d  d	S   jd7  _t
||d	S d
|dkr$  jd7  _t
||d	S |dkrX  jd7  _t|jj| d	S |dkr|	d|d r  jd7  _t|jj|d  d	S t||d}|dkr| _t|||| d	S |dkr(|	d|d r  jd7  _t
|dd	S   jd7  _t
|dd	S |dkr|	d|d r`  jd7  _t
|dd	S |	d|d r  jd7  _t
|dd	S   jd7  _t
|dd	S |dkr j||dS t||d}|dkr|kr | _t|||| d	S || }|dkrJ|d kr< _t||| d	S  j||dS |dks^|dkrt||d d}	|	dk rjd|d d |	 _|dkrt||||	 d	S t||||	 d	S | _t|||| d	S j|d q|W |  d S )Nc                  s0   s&dkr   nj d d d  _d S )Nr   r   )r@   )rH   r@   r   )r   rm   r   rj   r   r   return_none   s     
z%ssl_iterator.get.<locals>.return_noner   r   )r@   z=<>=r!   )r   r   r   z(),:+-rS   x)rM   ri   *z**/z//z/=.)i_fldi_dot)rM   ri   rj   edzInvalid floating-point literal)rC   r@   )r   rM   rj   rN   rk   popr@   rL   find
startswithra   r`   stringsstring_indicesindexr\   r   rZ   _ssl_iterator__after_dotr   r[   r]   r   rG   r^   rH   )
r   rm   rM   rt   resultr   cjZcjkr   )r   rm   r   rj   r   rn      s    










zssl_iterator.getc             C   sn  | j }| j}|d |kr&|jd|d |j}|j}||d  }|dkr|d|d sd|j|d d t||dd	}||kr|d
 | _|S | j	|d
 |f || _t
||||| d	S |dkr6|d|d s|j|d d t||dd	}||kr|d | _|S | j	|d |f || _t
||||| d	S |dksJ|dkr|dkr|d|d rt||dd	}||kr|d | _|S | j	|d |f || _t
||||| d	S |d|d rt||dd	}||kr|d
 | _|S | j	|d
 |f t
||||| d	S ||kr4|j|d d t||d |d}	|	dk r`|j|d d |	| _|dkrt|||||	 d	S t|||||	 d	S |dkr|d|d r||kr|j|d d |d | _t||dd	S |dkr4|d|d r||kr|j|d d |d | _t||dd	S |dkr\|d |d r||krh|j|d d |d
 | _t||d!d	S |d"|d rt||d#d	}||kr|d | _|S | j	|d |f || _t
||||| d	S |d$|d rLt||d%d	}||kr|d | _|S | j	|d |f || _t
||||| d	S |j|d d |d&ksp|d'kr|d(|d s|d"|d s|j|d d t|||||d  d	}||kr|d | _|S | j	|d |f || _t
||||| d	S t||d |d}	|	dk r.|j|d d |	| _|d|d | jdk rTt}
nt}
|
|||||	 d	S ))Nr   z'Expression unexpectedly ends with a dot)rC   r@   aznd.r!   )r@   z.and.)r   r   r   r'   ozr.z.or.r%   r|   r}   zq.z.eq.zqv.z.eqv.)rM   ri   rj   r   fzalse.r+   z.false.tzrue.r)   z.true.nzot.z.not.ze.z.ne.zeqv.z.neqv.glzt.)r   rj   rG   rM   r   rH   ra   r@   rk   appendr[   r   r^   r]   r_   r   r   )r   rz   r{   r   rj   rM   Zcswr   rq   r   tk_typer   r   r   Z__after_dot   s    























 


 zssl_iterator.__after_dotc             C   sJ   t | jdkr| jd d S | j}| j|d}| j| j|f || _|S )Nr   r   )rm   )rN   rk   r@   rn   r   )r   rm   Zprev_irq   r   r   r   
look_aheadT  s    zssl_iterator.look_aheadc                s   |j  j kstd|g} fdd}||    }|jddsT|jd ||    }|jdds|jd || t j |j|dS )	NzInvalid complex number literalc                 sR      } d }|  r6| jdkr*| jd | }   } |  sJ| jd || fS )N)r;   r<   )rC   )rn   r,   r   rG   r9   )rq   Zsign_tok)r   invalid_messager   r   get_parta  s    
z2ssl_iterator.get_complex_literal.<locals>.get_part,)r   )rC   ))r   r   r   )r   AssertionErrorr   rn   r:   rG   rb   r   )r   opening_tokenr   r   rq   r   )r   r   r   get_complex_literal]  s    
z ssl_iterator.get_complex_literalc             C   s    | j dd}|d kr|  |S )NT)rm   )rn   rQ   )r   r   rq   r   r   r   get_inside_parenthesesw  s    z#ssl_iterator.get_inside_parenthesesc             C   s   d}x| j | d}|d krP | r0|| nR|s>|  nD|dsP|  |   }| sh|  |   }|ds|  |d7 }| j dd}|d krP |jdds|  d}qW |S )Nr   )rm   rx   r   Tr   )r   )rn   r   rH   r:   )r   callbackone_requiredenable_commonZn_listrq   r   r   r   #collect_comma_separated_identifiers}  s2    



z0ssl_iterator.collect_comma_separated_identifiersTr   c             C   sB  t | j| jg d}|jj}d }d}	d }
|}x| j|d}d}|d kr|d k	rV|  t|jdkrx|d krpd S |  || d S |	 r2|j}|dkr|d kr|
  |
d kr|dkr|  n|
|	kr|  t|jdkr|d kr|
S |  || |
S |dkrpt|jdkr(|  || |	d7 }	|d k	rP|	|krPd S t | j| jg d}|jj}|}n|d	krg }| j|j|t|dkd
d krt}nt}|||j|j|d np|dkr(|d ks|dks|	dks|
d k	st|jdks|jd  s|  |	}
|| n|| q.|| q.W d S )N)r   r   r   r   )rm   Tr   r!   r   r   ()r   r   enable_implied_doru   )rc   r   r@   r   r   rn   rQ   rN   rH   r,   rR   #collect_comma_separated_expressionsintrd   re   r   r   )r   r   r   first_get_optional!stop_after_given_number_of_commasr   tokenstappZ
last_commaZn_callbacksZi_assignment_opZget_optionalrq   tvnested_tokensr   r   r   r   r     s    













z0ssl_iterator.collect_comma_separated_expressionsc             C   s.   g }| j |j|d |t|j|j|d d S )N)r   r   )r   r   r   )r   r   rd   r   r   )r   r   r   r   r   r   r   collect_to_matching_parenthesis  s    z,ssl_iterator.collect_to_matching_parenthesisc             C   s(   g }| j |j|dd t| j|j|dS )Nr!   )r   r   r   )r   r   r   )r   r   re   r   r   )r   r   r   r   r   r   get_implied_do  s    zssl_iterator.get_implied_do)N)F)F)FF)NTNr   )rV   rW   rX   __doc__rY   r   rl   rr   rs   rn   r   r   r   r   r   r   r   r   r   r   r   r   rh   u   s&   

[l
	 
   
Hrh   c                s*   g  x | D ] fdd}|  q
W  S )Nc                 sJ     r@j} t| dkr@| d }| r@ t|jd d S  S )Nr   r   )r   )r0   r   rN   r2   extendremove_redundant_parenthesesr   )r   Z	inner_tok)r   rq   r   r   handle_seq_with_one_parentheses  s    zEremove_redundant_parentheses.<locals>.handle_seq_with_one_parenthesesr   )r   r   r   )r   rq   r   r     s
    


r   c             C   sP  g }d}t | }x8||k rJ| | }|d7 }|jddr>t |dkrP|  |t | krd|  |d  rt |dkr|d  rt|d j|d j|dd  d}|  n|d }|  | | }|d7 }| r |t | k r | |  r | | }|d7 }t|j|j||gd}|	t
|j|j||gd q|	| qW |S )Nr   r   z**)r   )r   r   r   )rN   r:   rH   r2   r   rc   r   r   r~   r   rf   )r   r   i_tokZn_toksrq   Zbase_tokZexponent_toknext_tokr   r   r   group_power
  sL     
 
 r   c               @   s   e Zd ZdddgZdd ZdS )implied_do_info
dlist_sizeid_tok
fls_tokensc                s`   t dkst fdd}|ddr: jd  n"|ddsHt jdd   d S )Nr#   c                s   |  }|  sdS |j}t|dk r*dS |d  s:dS |d jddsNdS t|   _|d  _t|d j|d j	|dd  dg _
d	S )
NFr#   r   r   ru   )r   r!   )r   r   r   T)r0   r   rN   r   r:   r   r   rc   r   r   r   )r@   rq   r   )r   r   r   r   get_first_tokens7  s        
&z2implied_do_info.__init__.<locals>.get_first_tokensr   )r@   r   )rN   r   r   r   r   )r   r   r   r   )r   r   r   r   5  s    
zimplied_do_info.__init__N)rV   rW   rX   rY   r   r   r   r   r   r   1  s   
r   c             C   s    t | jdko| jd jddS )Nr   r   rw   )r   )rN   r   r:   )tok_seqr   r   r   tok_seq_is_starH  s    r   Nc             C   s  |d krg }x| D ]}|  rD|d t|j|d |d q| rp|d t|j|d |d q| r|d t|j|d |d q| r||  q| r|d t|j|d |d q|	 r|d	|  d  q||j qW |S )
N[)r   r   ]r   r   {}zpower(zformat()
r0   r   tokens_as_stringsr   r2   r4   r*   rU   r6   r8   )r   r   rq   r   r   r   r   K  s2    




r   c             C   s   d t| dS )N )r   )joinr   )r   r   r   r   tokens_as_stringg  s    r   Tc             C   s   |d k}|rg }x| D ]}|  r:t|j|dd krd S q| rn|d t|j|dd krbd S |d q| rzd S | rd S | rd S | rd S |s|j	ddrd S ||j qW |rd
|S |S )N)r   r   r   r   z**)r   r   )r0   tokens_as_python_coder   r2   r   r4   r*   r6   r8   r:   r   )r   r   allow_powerZjoin_resultrq   r   r   r   r   j  s4    


r   c             C   st   xnt |D ]b\}}| r,t| |j|d q
| r
|sB| | q
d }|d }|t|krb|| }| || q
W d S )N)r   r   with_next_tokenr   )	enumerater?   search_for_id_tokensr   r   rN   )r   r   r   r   rq   r   Zj_tokr   r   r   r     s    
r   c             C   s    |d krg }t |j| dd |S )NF)r   r   r   )r   r   )r   r   r   r   r   extract_identifiers  s
     r   c             C   sx   xr|D ]j}|  stt|jdkr&q|jd }| rD| |d q| rt|jd}t| |jd |j d qW d S )Nr   )rq   )r   )r   r   )	r0   r   rN   r   r   r4   r   %search_for_data_or_read_target_tokensr   )r   r   rq   Zfirst_target_tokZidir   r   r   r     s    

r   c               @   s@   e Zd ZdZddgZdd Zdd Zdd	 Zd
d Zdd Z	dS )fss_iteratorzformat string stripped iteratorfssr@   c             C   s   || _ d| _d S )Nr   )r   r@   )r   r   r   r   r   r     s    zfss_iterator.__init__c             C   s   | S )Nr   )r   r   r   r   rl     r   zfss_iterator.__iter__c             C   s   |   }|d krt|S )N)rn   ro   )rp   rq   r   r   r   rr     s    zfss_iterator.__next__c             C   s   |   S )N)rr   )rp   r   r   r   rs     s    zfss_iterator.nextc                s   j j}t| jd k	s"t fdd}x jk rֈ j}|| }|dkrh  jd7  _q6|dkr  jd7  _t|ddS d|d	kr  jd7  _t||dS |d
kr  jd7  _t|j	j
| dS |dks|dkrLt||d d}|d	k s$|d|s*|  |d  _t||| j dS t||d}|d	kr| _|d jrrj jd\}}|j|  dkst|d7 }t||| }|| }	|j||	 }
t|
|krtj|dd|
d
d	ks|
dd	kr$tj|ddx> jk rbj jd\}}||	krPP   jd7  _q&W t||
dS |d js|d jr  jd7  _t||| j dS t|||| dS d|d	krNt||d d}|d	kr6| _|d|r6t||d d}|d	k r0|  | _t||| j dS |dksb|dkr  jd7  _t||d d}|d	kr| _t||| j dS |d|s|d|r  jd7  _t||| j dS |dkrJ  jd7  _|d js$|d jr2  jd7  _t||| j dS |dkrΈ  jd7  _|d js|d jr  jd7  _t| jd}|d	k r|  | _t||| j dS |  q6W d  _d S )Nc                  s   j dtd  jd d S )NzInvalid FORMAT specificationr   )rC   r@   )rG   minr@   r   )r   r   rj   r   r   raise_invalid  s    z'fss_iterator.get.<locals>.raise_invalidr   r   rv   Z1x)r   r   r   z():/$r   rS   r;   r<   )rM   ri   rj   ph)r@   zRFATAL: Not supported: FORMAT Hollerith edit descriptor spanning continuation lines)r@   rC   "zBFATAL: Not supported: FORMAT Hollerith edit descriptor with quotesZdefgizry   r   r   Zbnbzr!   sr   r)r   rM   rN   r@   r   rg   r   ra   r`   r   r   r   r   r   text_locationtextlowerr   rJ   rE   r[   )r   rM   r   r   r   r   slZi_textZnhZj_textr   r   )r   r   rj   r   rn     s    


 

 



 
 

zfss_iterator.getN)
rV   rW   rX   r   rY   r   rl   rr   rs   rn   r   r   r   r   r     s   r   c             C   s<   t | dkst| d }t | dks,| s8|jdd |S )Nr   r   zInvalid statement label)rC   )rN   r   r    rG   )r   rq   r   r   r   get_statement_label_token$  s
    r   r   c                s  g  fdd}x| D ]}|  r2|j q| rbd|jdd d    q| r|j   q| r|j}|dks|dkr| r  | |dkr  q|   qtqW | r  d	S )Nc                  s   t dkod  kS )Nr   r   )rN   r   )commar   r   r   result_ends_with_comma-  s    z4fmt_tokens_as_string.<locals>.result_ends_with_commarS   z''r   r   rB   )
r    r   r   r*   rT   r8   r,   r~   r   r   )r   r   r   rq   r   r   )r   r   r   fmt_tokens_as_string+  s2    

 
r   )N)NT)N)r   )'
__future__r   r   r   fabler   r   r   r   objectr	   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   sB   ^   '


{