3
Ud$                 @   s   d Z ddlmZmZ ddlZddlZddlZddlj	j
ZddlmZ ddljZddlmZ ddlmZ ejjejjddd d!d"gejjdddgejjdddgdd Zejdd Zejdd Zejjdej G dd dZdS )#zl
Tests parsers ability to read and parse non-local files
and hence require a network connection to be read.
    )BytesIOStringION)	DataFrame)read_feather)read_csvzcompress_type, extensiongzip.gzbz2.bz2zip.zipxz.xzmodeexplicitinferenginepythoncc             C   s   t | |||| d S )N)check_compressed_urls)salaries_tablecompress_type	extensionr   r    r   W/var/www/html/virt/lib64/python3.6/site-packages/pandas/tests/io/parser/test_network.pytest_compressed_urls   s    r   c             C   s8   d}|| }|dkr|}t |d||d}tj||  d S )NzXhttps://github.com/pandas-dev/pandas/raw/master/pandas/tests/io/parser/data/salaries.csvr   	)sepcompressionr   )r   tmassert_frame_equal)r   r   r   r   r   base_urlurlZ	url_tabler   r   r   r      s    r   c             C   s   t | ddddS )z DataFrame with the tips dataset.iodatacsvztips.csv)r   )Zdatapathr   r   r   tips_df1   s    r&   s3_resourcec               @   s   e Zd Zejddd Zdd Zdd Zdd	 Zd
d Z	dd Z
dd Zdd Zdd Zdd Zdd Zejddd Zdd Zdd Zdd  Zejdd!d" Zd#S )$TestS3s3fsc             C   sz   xFdD ]>\}}t d| |d}t|ts,t|j s8ttj|| qW t d}t|ts^t|j sjttj|| d S )N .gzr   .bz2r	   zs3://pandas-test/tips.csv)r   zs3://cant_get_it/tips.csvr*   Nr+   r   r,   r	   )r-   r.   r/   )r   
isinstancer   AssertionErroremptyr   r    )selfr&   extcompdfr   r   r   test_parse_public_s3_bucket:   s    z"TestS3.test_parse_public_s3_bucketc             C   s@   t ddd}t|tst|j s&ttj|jd d | d S )Nzs3n://pandas-test/tips.csv
   )nrows)r   r0   r   r1   r2   r   r    iloc)r3   r&   r6   r   r   r   test_parse_public_s3n_bucketK   s    z#TestS3.test_parse_public_s3n_bucketc             C   s@   t ddd}t|tst|j s&ttj|jd d | d S )Nzs3a://pandas-test/tips.csvr8   )r9   )r   r0   r   r1   r2   r   r    r:   )r3   r&   r6   r   r   r   test_parse_public_s3a_bucketS   s    z#TestS3.test_parse_public_s3a_bucketc             C   sX   xRdD ]J\}}t d| d|d}t|ts.t|j s:ttj|jd d | qW d S )Nr*   .gzr   .bz2r	   zs3://pandas-test/tips.csvr8   )r9   r   r*   Nr=   r   r>   r	   )r?   r@   rA   )r   r0   r   r1   r2   r   r    r:   )r3   r&   r4   r5   r6   r   r   r   !test_parse_public_s3_bucket_nrowsZ   s
    z(TestS3.test_parse_public_s3_bucket_nrowsc       	      C   s   d}xdD ]~\}}t d| ||d}|j|ks2txTdD ]L}|j }t|tsRt|j s^t|j|| ||d
   }tj	|| q8W q
W d S )N   r*   .gzr   .bz2r	   zs3://pandas-test/tips.csv)	chunksizer   r         r*   NrD   r   rE   r	   )rI   rJ   rK   )r   rG   rH   )
r   rF   r1   	get_chunkr0   r   r2   r:   r   r    )	r3   r&   rF   r4   r5   	df_readeri_chunkr6   true_dfr   r   r   #test_parse_public_s3_bucket_chunkeda   s    
z*TestS3.test_parse_public_s3_bucket_chunkedc       	      C   s   d}xdD ]\}}t d| ||dd	}|j|ks4txTdD ]L}|j }t|tsTt|j s`t|j|| ||d   }tj	|| q:W q
W d S )NrC   r*   .gzr   .bz2r	   zs3://pandas-test/tips.csvr   )rF   r   r   r   rG   rH   r*   NrQ   r   rR   r	   )rS   rT   rU   )r   rG   rH   )
r   rF   r1   rL   r0   r   r2   r:   r   r    )	r3   r&   rF   r4   r5   rM   rN   r6   rO   r   r   r   *test_parse_public_s3_bucket_chunked_pythonr   s    
z1TestS3.test_parse_public_s3_bucket_chunked_pythonc             C   sN   xHdD ]@\}}t d| d|d}t|ts.t|j s:ttj|| qW d S )Nr*   .gzr   .bz2r	   zs3://pandas-test/tips.csvr   )r   r   r*   NrW   r   rX   r	   )rY   rZ   r[   )r   r0   r   r1   r2   r   r    )r3   r&   r4   r5   r6   r   r   r   "test_parse_public_s3_bucket_python   s    z)TestS3.test_parse_public_s3_bucket_pythonc             C   sJ   xDdD ]<}t d| ddd}t|ts*t|j s6ttj|| qW d S )	Nr*   .gz.bz2zs3://pandas-test/tips.csvr   r   )r   r   )r*   r]   r^   )r   r0   r   r1   r2   r   r    )r3   r&   r4   r6   r   r   r   test_infer_s3_compression   s    
z TestS3.test_infer_s3_compressionc             C   sZ   xTdD ]L\}}t d| dd|d	}t|ts0t|j s<ttj|jd d | qW d S )Nr*   .gzr   .bz2r	   zs3://pandas-test/tips.csvr   r8   )r   r9   r   r*   Nr`   r   ra   r	   )rb   rc   rd   )r   r0   r   r1   r2   r   r    r:   )r3   r&   r4   r5   r6   r   r   r   (test_parse_public_s3_bucket_nrows_python   s    z/TestS3.test_parse_public_s3_bucket_nrows_pythonc             C   s@   t jt td W d Q R X t jt td W d Q R X d S )Nzs3://nyqpug/asdf.csvzs3://cant_get_it/file.csv)pytestraisesIOErrorr   )r3   r   r   r   test_read_s3_fails   s    zTestS3.test_read_s3_failsc          	   C   s<   dd l }t|jjf}tj|dd |jd W d Q R X d S )Nr   z#The specified bucket does not exist)matchz/s3://an_s3_bucket_data_doesnt_exit/not_real.csv)botocoreFileNotFoundError
exceptionsClientErrorrf   rg   to_csv)r3   r&   rk   errorr   r   r   test_write_s3_csv_fails   s    zTestS3.test_write_s3_csv_failsZpyarrowc          	   C   s<   dd l }t|jjf}tj|dd |jd W d Q R X d S )Nr   z#The specified bucket does not exist)rj   z3s3://an_s3_bucket_data_doesnt_exit/not_real.parquet)rk   rl   rm   rn   rf   rg   Z
to_parquet)r3   r&   rk   rp   r   r   r   test_write_s3_parquet_fails   s    z"TestS3.test_write_s3_parquet_failsc             C   s\   |j jjddd}tt|d j dd}t|ts8t|j	 sDtt|}t
j|| d S )Nzpandas-testztips.csv)BucketKeyBodyutf8)encoding)metaclientZ
get_objectr   r   readr0   r   r1   r2   r   r    )r3   r'   Z	tips_fileZ	s3_objectresultexpectedr   r   r   $test_read_csv_handles_boto_s3_object   s    
z+TestS3.test_read_csv_handles_boto_s3_objectc             C   s   dd l }ttjjddtdd}t }|j| t|j	 j
d}|jdjd|d	 |jj  |jtjd
d* tddd ddd |jD kstW d Q R X d S )Nr   i    Zabcd)columnszutf-8zpandas-testzlarge-file.csv)rt   ru   r)   )loggerzs3://pandas-test/large-file.csvrC   )r9     T c             s   s   | ]}|j dd V  qdS )rH   N)args).0xr   r   r   	<genexpr>   s    z8TestS3.test_read_csv_chunked_download.<locals>.<genexpr>)r   r   )r)   r   nprandomZrandnlistr   ro   r   getvalueencoders   Z
put_objectZS3FileSystemZclear_instance_cacheZat_levelloggingDEBUGr   recordsr1   )r3   r'   Zcaplogr)   r6   Zstr_bufbufr   r   r   test_read_csv_chunked_download   s    

z%TestS3.test_read_csv_chunked_downloadc             C   s   t d}tj|| d S )Nzs3://pandas-test/tips#1.csv)r   r   r    )r3   r&   r{   r   r   r   test_read_s3_with_hash_in_key   s    z$TestS3.test_read_s3_with_hash_in_keyc             C   s    t |}t d}tj|| d S )Nz's3://pandas-test/simple_dataset.feather)r   r   r    )r3   Zfeather_filer|   resr   r   r   test_read_feather_s3_file_path   s    z%TestS3.test_read_feather_s3_file_pathN)__name__
__module____qualname__tdZ
skip_if_nor7   r;   r<   rB   rP   rV   r\   r_   re   ri   rq   rr   r}   r   r   r   r   r   r   r   r(   7   s    			r(   )r   r   )r	   r
   )r   r   )r   r   )__doc__r#   r   r   r   Znumpyr   rf   Zpandas.util._test_decoratorsutilZ_test_decoratorsr   Zpandasr   Zpandas._testingZ_testingr   Zpandas.io.feather_formatr   Zpandas.io.parsersr   marknetworkZparametrizer   r   Zfixturer&   ZusefixturesZskip_if_not_us_localer(   r   r   r   r   <module>   s&   
 
