B lc0@sddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZmZmZddlmZddlZddlZddlmZddlmZddlZejdkrddlZd d Zd d Z n0ejd krddl!Z!dd Zdd Z n dd ZeZ ej"ddZ#ddddddZ$dddddZ%d/ddddddd d!d"Z&d#d$d%d&d'Z'dd(d)d*d+d,Z(Gd-d.d.Z)dS)0) annotationsN)Path)AnyOptionalUnion)urlparse) HTTPAdapter)RetryposixcCst|tjdS)N)fcntllockfZLOCK_EX) file_handler2lib/python3.7/site-packages/dials_data/download.py_platform_locksrcCst|tjdS)N)r r ZLOCK_UN)r rrr_platform_unlocksrntc Cs`|dxPyt|tjdPWq tk rV}z|jtjkrFWdd}~XYq Xq WdS)Nr)seekmsvcrtlockingfilenoZLK_LOCKOSErrorerrnoZEDEADLK)r errrr#s  cCs"|dt|tjddS)Nrr)rrrrZLK_UNLCK)r rrrr/s cCs tddS)Nz+File locking not supported on this platform)NotImplementedError)r rrrr5sccs.d}zt|d}dVWd|r(t|XdS)z Cross-platform file locking. Open a file for writing or appending. Then a file lock can be obtained with: with open(filename, 'w') as fh: with _file_lock(fh): (..) FTN)rr)r lockrrr _file_lock;s  rzrequests.Sessionstrr)sessionurlpyfilec Csn|j|ddV}||jjddd|jdd&}x|jddD]}||qDWWdQRXWdQRXdS) z+ Downloads a single URL to a file. T)stream)parentsexist_okwb)modei)Z chunk_sizeN)getZraise_for_statusparentmkdiropenZ iter_contentwrite)rr r!rfchunkrrr_download_to_fileOs r/) file_to_hashreturnc sLt}|d,x$tfdddD]}||q(WWdQRX|S)z$Returns the SHA256 digest of a file.rbcs dS)Ni)readr)r-rr_zfile_hash..r5N)hashlibZsha256r*iterupdateZ hexdigest)r0Z sha256_hashblockr)r-r file_hash[s  r:FTboolzUnion[bool, dict[str, Any]])ignore_hashinfoverify read_onlyverbosepre_scanr1c s*|tjjkrdStjj|}tj||r:s:dS|d}|rL|rXtj|}d|krvdd|dD|d<fddt|d|dD}|s|rtdd |Drd S|rdSj d d d  d |j d d"} t | t |} WdQRXWdQRX|rt| sdS| |d<|S)aCheck for the presence or integrity of the local copy of the specified test dataset. If the dataset is not available or out of date then attempt to download/update it transparently. :param pre_scan: If all files are present and all file sizes match then skip file integrity check and exit quicker. :param read_only: Only use existing data, never download anything. Implies pre_scan=True. :param verbose: Show everything as it happens. :param verify: Check all files against integrity information and fail on any mismatch. :returns: False if the dataset can not be downloaded/updated for any reason. True if the dataset is present and passes a cursory inspection. A validation dictionary if the dataset is present and was fully verified. Fhashinfor=cSsg|]}iqSrr).0_rrr sz!fetch_dataset..datac s>g|]6\}}|dtjt|dj|d|dqS)r files)r filerFr=)ospathbasenamerr')rBsourcerA) target_dirrrrDscssB|]:}|do8|ddo8|dd|djkVqdS)rGr=sizeN)is_filer'statst_size)rBitemrrr sz fetch_dataset..T)r#r$z.lock.w)r&N) dials_datadatasets definitionrepository_locationis_dirr'Zcreate_integrity_recordzipallr)Z with_namer*r_fetch_filelist) Zdatasetr<r=r>r?r@rVZintegrity_infofilelistZfhZverification_recordsr)rLr fetch_datasetds8        r]zlist[dict[str, Any]]zlist[dict[str, Any] | None])r\r1c Csxtf}ttddddddddd hd d }|d ||d |tjjdd}|t t ||}t |SQRXdS)NrTiiiiii)ZtotalZbackoff_factorZraise_on_statusZstatus_forcelist)Z max_retrieszhttp://zhttps://) max_workers) requestsZSessionrr Zmount concurrentZfuturesZThreadPoolExecutormap functoolspartial _fetch_filelist)r\ZrsZ retry_adapterZpoolresultsrrrr[s   r[zdict[str, Any]zdict[str, Any] | None)rrKr1c s|d}|drXd}|drX|dd|djkr.zDecompressing z.zip)rIzExpected file z not present in zip archive z not present in tar archive )rNrOrPr:printr/r(rZsuffixzipfileZZipFileextractKeyErrortarfiler*)rrKZvalidZ downloadedZvalidation_recordZzfr-Ztarr)rLrres\   (  "(,rec@sPeZdZdZdddZddddZd d Zddd d dZdddddZd S) DataFetcheraA class that offers access to regression datasets. To initialize: df = DataFetcher() Then df('insulin') returns a Path object to the insulin data. If that data is not already on disk it is downloaded automatically. To disable all downloads: df = DataFetcher(read_only=True) Do not use this class directly in tests! Use the dials_data fixture. FcCs,i|_tj|_|o$t|jtj|_dS)N) _cacherTrUrW _target_dirrHaccessW_OK _read_only)selfr>rrr__init__s zDataFetcher.__init__r)r1cCsd|jrdnd|jS)Nz<{}DataFetcher: {}>zR/O )formatrtrq)rurrr__repr__s zDataFetcher.__repr__cKs|S)z An overridable function to mangle lookup results. Used in tests to transform negative lookups to test skips. Overriding functions should add **kwargs to function signature to be forwards compatible. r)ruresultkwargsrrr result_filter%szDataFetcher.result_filterN) test_datacKsx||jkr|||j|<|dkr2tjdtdd|j|sH|jddS|sf|jtj|j|dS|j|j|dS)a Return the location of a dataset, transparently downloading it if necessary and possible. The return value can be manipulated by overriding the result_filter function. :param test_data: name of the requested dataset. :param pathlib: Whether to return the result as a Python pathlib object. The default for this setting is 'False' for now (leading to a py.path.local object being returned), but the default will change to 'True' in a future dials.data release. Set to 'True' for forward compatibility. :return: A pathlib or py.path.local object pointing to the dataset, or False if the dataset is not available. NzThe DataFetcher currently returns py.path.local() objects. This will in the future change to pathlib.Path() objects. You can either add a pathlib=True argument to obtain a pathlib.Path() object, or pathlib=False to silence this warning for now.) stacklevelF)rz) rp_attempt_fetchwarningswarnDeprecationWarningr|pyrIZlocal)rur}pathlibr{rrr__call__.s   zDataFetcher.__call__zOptional[Path])r}r1cCs:|jrt|ddd}nt|ddd}|r2|j|SdSdS)NT)r@r>F)rtr]rq)rur}Zdata_availablerrrrNs zDataFetcher._attempt_fetch)F)N) __name__ __module__ __qualname____doc__rvryr|rrrrrrro s    ro)FFFFT)*Z __future__rconcurrent.futuresra contextlibrrcr6rHrnrrkrrtypingrrrZ urllib.parserZpy.pathrr`Zrequests.adaptersrZurllib3.util.retryr Zdials_data.datasetsrTnamer rrrcontextmanagerrr/r:r]r[rerorrrrsL           HF