3 qdJH @sbdZddlZddlmZddlZddlmZmZmZddl Z ddl Z ddl Z ddl m Z mZmZmZmZmZmZmZmZmZmZddlmZmZmZmZmZddlZddl m!Z!ddl"m#Z#m$Z$dd l%m&Z&dd l'm(Z(e$Z)e*eeeZ+e+j,d er dd lm-Z-e.d ddZ/e!ee!edddZ0dd ddZ1e!ee!edddZ2ddZ3e!e.dddZ4d6e!ee5ee5ee5eee5efdddZ6e5e5dd d!Z7d"d#d$d%d&Z8eee5ee5e5ffeee5ee5e5ffd'd(d)Z9e!ee5ee5d*d+d,Z:d7e5eee5ee5effe.e.d/d0d1Z;Gd2d3d3ejZ?dS)8zCommon IO api utilitiesN)abc)BufferedIOBaseBytesIO RawIOBase) IO TYPE_CHECKINGAnyAnyStrDictListMappingOptionalTupleTypeUnion)urljoinurlparse uses_netloc uses_params uses_relative)FilePathOrBuffer)_get_lzma_file _import_lzma)import_optional_dependency) is_file_like)IOBase)returncCst|tsdSt|jtkS)z Check to see if a URL has a valid protocol. Parameters ---------- url : str or unicode Returns ------- isurl : bool If `url` has a valid protocol return True otherwise False. F) isinstancestr parse_urlscheme _VALID_URLS)urlr$2/tmp/pip-build-7vycvbft/pandas/pandas/io/common.pyis_url1s r&)filepath_or_bufferrcCst|trtjj|S|S)a] Return the argument with an initial component of ~ or ~user replaced by that user's home directory. Parameters ---------- filepath_or_buffer : object to be converted if possible Returns ------- expanded_filepath_or_buffer : an expanded filepath or the input if not expandable )rrospath expanduser)r'r$r$r% _expand_userCs  r+cCst|trtddS)NzPassing a bool to header is invalid. Use header=None for no header or header=int or list-like of ints to specify the row(s) making up the column names)rbool TypeError)headerr$r$r%validate_header_argXs r/cCs.t|dr|jSt|tjr&t|St|S)ad Attempt to convert a path-like object to a string. Parameters ---------- filepath_or_buffer : object to be converted Returns ------- str_filepath_or_buffer : maybe a string version of the object Notes ----- Objects supporting the fspath protocol (python 3.6+) are coerced according to its __fspath__ method. For backwards compatibility with older pythons, pathlib.Path and py.path objects are specially coerced. Any other object is passed through unchanged, which includes bytes, strings, buffers, or anything else that's not even path-like. __fspath__)hasattrr0rpathlibPathrr+)r'r$r$r%stringify_pathas   r4cOsddl}|jj||S)z` Lazy-import wrapper for stdlib urlopen, as that imports a big chunk of the stdlib. rN)urllib.requestrequesturlopen)argskwargsurllibr$r$r%r7sr7)r#rcCst|tod|ko|jd S)zR Returns true if the given URL looks like something fsspec can handle z://http://https://)r;r<)rr startswith)r#r$r$r% is_fsspec_urls r>)r'encoding compressionmodestorage_optionscCst|}t|tr\t|r\t|}|jjdd}|dkrAssertionErrorr=replacerZbotocore.exceptionsrErFPermissionError ImportErroropentupledictbytesmmapr+rtype ValueError)r'r?r@rArBreqcontent_encodingreaderrDZerr_types_to_retry_with_anonrErFZfile_objmsgr$r$r%get_filepath_or_buffersV         $  (  r[)r)rcCsddlm}td||S)z converts an absolute native path to a FILE URL. Parameters ---------- path : a path in native format Returns ------- a valid FILE URL r) pathname2urlzfile:)r5r\r)r)r\r$r$r%file_path_to_urls r]z.gzz.bz2z.zipz.xz)rCbz2zipxz)r@rcCs\t|trPt|}y|jd}WqTtk rL}ztd|WYdd}~XqTXni}||fS)a Simplifies a compression argument to a compression method string and a mapping containing additional arguments. Parameters ---------- compression : str or mapping If string, specifies the compression method. If mapping, value at key 'method' specifies compression method. Returns ------- tuple of ({compression method}, Optional[str] {compression arguments}, Dict[str, str]) Raises ------ ValueError on mapping missing 'method' key methodz.If mapping, compression must have key 'method'N)rr rRpopKeyErrorrV)r@compression_argserrr$r$r%get_compression_method s rf)r'r@rcCs|dkr dS|dkrVt|}t|ts*dSx&tjD]\}}|jj|r4|Sq4WdS|tkrb|Sd|}ddgtt}|d|7}t|dS)a Get the compression method for filepath_or_buffer. If compression='infer', the inferred compression method is returned. Otherwise, the input compression method is returned unchanged, unless it's invalid, in which case an error is raised. Parameters ---------- filepath_or_buffer : str or file handle File path or object. compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None} If 'infer' and `filepath_or_buffer` is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no compression). Returns ------- string or None Raises ------ ValueError on invalid compression specified. NZinferzUnrecognized compression type: z Valid compression types are ) r4rr_compression_to_extensionitemslowerendswithsortedrV)r'r@ extensionrZZvalidr$r$r%infer_compression.s   rmFT)rAr@ memory_mapis_textcCsyddlm}tt|f}Wntk r6ttf}YnXt} |} t|}t|t} t |\}} | rnt ||}|r|dkr| rt j ||f| } nt j fd|i| } n|dkr| rtj||f| } ntj|f| } n|dkrjt||f| } | j| | jdkr | } n^| jdkr| j}t|d kr<| j |j} n,t|dkrZtd |ntd |n,|d krtt||} nd |}t|| j| nL| r|rt ||||dd} n"|rt ||ddd} n t ||} | j| |r@|st| |r@ddlm}|| ||dd}t| ttfs<| j||} |rt| dryt| }| j|} Wntk rYnX| | fS)af Get file handle for given path/buffer and mode. Parameters ---------- path_or_buf : str or file handle File path or object. mode : str Mode to open path_or_buf with. encoding : str or None Encoding to use. compression : str or dict, default None If string, specifies compression mode. If dict, value at key 'method' specifies compression mode. Compression mode must be one of {'infer', 'gzip', 'bz2', 'zip', 'xz', None}. If compression mode is 'infer' and `filepath_or_buffer` is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no compression). If dict and compression mode is one of {'zip', 'gzip', 'bz2'}, or inferred as one of the above, other entries passed as additional compression options. .. versionchanged:: 1.0.0 May now be a dict with key 'method' as compression mode and other keys as compression options if compression mode is 'zip'. .. versionchanged:: 1.1.0 Passing compression options as keys in dict is now supported for compression modes 'gzip' and 'bz2' as well as 'zip'. memory_map : boolean, default False See parsers._parser_params for more information. is_text : boolean, default True whether file/buffer is in text format (csv, json, etc.), or in binary mode (pickle, etc.). errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. .. versionadded:: 1.1.0 Returns ------- f : file-like A file-like object. handles : list of file-like objects A list of file-like object that were opened in this function. r)S3FilerCfileobjr^r_wrzZero files found in ZIP file z9Multiple files found in ZIP file. Only one file per ZIP: r`zUnrecognized compression type: r)r?errorsnewlinerM)rurv) TextIOWrapperfileno)Zs3fsrprrrOlistr4rrrfrmrCrPGzipFiler^BZ2File _BytesZipFileappendrAnamelistlenrbrVrlzmaiorwr1 _MMapWrapperrK Exception)Z path_or_bufrAr?r@rnrorurpZneed_text_wrappingZhandlesfZis_pathrdzfZ zip_namesrZrwgwrappedr$r$r% get_handleds~=               rcsHeZdZdZd eeeedfdd ZfddZe dd Z Z S) r|a  Wrapper for standard library class ZipFile and allow the returned file-like handle to accept byte strings via `write` method. BytesIO provides attributes of file-like object and ZipFile.writestr writes bytes strings into a member of the archive. N)filerA archive_namec s4|dkr|jdd}||_tj||tjf|dS)NwbrGbr)rrG)rMrsuper__init__zipfile ZIP_DEFLATED)selfrrArr9) __class__r$r%rs z_BytesZipFile.__init__cs(|j}|jdk r|j}tj||dS)N)filenamerrwritestr)rdatar)rr$r%write$s z_BytesZipFile.writecCs |jdkS)N)fp)rr$r$r%closed*sz_BytesZipFile.closed)N) __name__ __module__ __qualname____doc__rrr rrpropertyr __classcell__r$r$)rr%r|s r|c@sHeZdZdZedddZedddZddd d Zedd d Z d S)ra Wrapper for the Python's mmap class so that it can be properly read in by Python's csv.reader class. Parameters ---------- f : file object File object to be mapped onto memory. Must support the 'fileno' method or have an equivalent attribute )rcCstj|jdtjd|_dS)Nr)access)rTrxZ ACCESS_READ)rrr$r$r%r<sz_MMapWrapper.__init__)namecCs t|j|S)N)getattrrT)rrr$r$r% __getattr__?sz_MMapWrapper.__getattr__)rcCs|S)Nr$)rr$r$r%__iter__Bsz_MMapWrapper.__iter__cCs$|jj}|jd}|dkr t|S)Nzutf-8r)rTreadlinedecode StopIteration)rZnewbytesrvr$r$r%__next__Es   z_MMapWrapper.__next__N) rrrrrrrrrrr$r$r$r%r/s  r)NNNN)NNFTN)@rr^ collectionsrrCrrrrrTr(r2typingrrrr r r r r rrr urllib.parserrr rrrrZpandas._typingrZ pandas.compatrrZpandas.compat._optionalrZpandas.core.dtypes.commonrrsetr"discardrr,r&r+r/r4r7r>rr[r]rgrfrmrZipFiler|Iteratorrr$r$r$r%sZ 4        0[" 7*$!