o c @sddlZddlZddlZddlmZddlmZddlmZddl m Z m Z Gddde Z Gdd d e ZGd d d e Zd d iddddd dddf ddZddZdddZd ddZd ddZddZedkrteejdddSdS)!N) OptionParser) DeferredList) deferToThread) iteritems networkStringc@s eZdZdS) FetchErrorN)__name__ __module__ __qualname__r r 5/usr/lib/python3/dist-packages/landscape/lib/fetch.pyr src@s$eZdZddZddZddZdS) HTTPCodeErrorcC||_||_dSN) http_codebody)selfrrr r r __init__ zHTTPCodeError.__init__cC d|jS)NzServer returned HTTP code %drrr r r __str__ zHTTPCodeError.__str__cCr)Nzrrr r r __repr__rzHTTPCodeError.__repr__N)rr r rrrr r r r r s r c@s0eZdZddZddZddZeddZd S) PyCurlErrorcCrr) error_code_message)rrmessager r r rrzPyCurlError.__init__cCd|j|jfS)Nz Error %d: %srrrr r r r#szPyCurlError.__str__cCr)Nzr rr r r r&szPyCurlError.__repr__cCs|jSr)rrr r r r*szPyCurlError.messageN)rr r rrrpropertyrr r r r rs rFiXTc  Csddl} t|ts|d}t|} t}|dur| }|| jt t ||rF|| j d|rF|| j t ||| j| j|rV|drV|| jt ||rh|| jddtt|D|rq|| jd| rz|| jd | dur|| jt | | dur|| jt | || jd || j||| jd || j||| jd || j|j|| j d|| j!d z|"Wn| j#y}z t$|j%d|j%d d}~ww|&}|'| j(}|d krt)|||S) aCRetrieve a URL and return the content. @param url: The url to be fetched. @param post: If true, the POST method will be used (defaults to GET). @param data: Data to be sent to the server as the POST content. @param headers: Dictionary of header => value entries to be used on the request. @param curl: A pycurl.Curl instance to use. If not provided, one will be created. @param cainfo: Path to the file with CA certificates. @param insecure: If true, perform curl using insecure option which will not attempt to verify authenticity of the peer's certificate. (Used during autodiscovery) @param follow: If True, follow HTTP redirects (default True). @param user_agent: The user-agent to set in the request. @param proxy: The proxy url to use for the request. rNzutf-8Tzhttps:cSsg|]}d|qS)z%s: %sr ).0pairr r r \szfetch..Fs gzip,deflate)*pycurl isinstancebytesencodeioBytesIOCurlsetoptURLrstrPOST POSTFIELDSIZElen READFUNCTIONread startswithCAINFO HTTPHEADERsortedrSSL_VERIFYPEERFOLLOWLOCATION USERAGENTPROXY MAXREDIRSCONNECTTIMEOUTLOW_SPEED_LIMITLOW_SPEED_TIMENOSIGNAL WRITEFUNCTIONwriteDNS_CACHE_TIMEOUTENCODINGperformerrorrargsgetvaluegetinfo HTTP_CODEr )urlpostdataheaderscainfocurlconnect_timeout total_timeoutinsecurefollow user_agentproxyr*outputinputerrr r r fetch/s\      r_cOsttg|Ri|S)z]Retrieve a URL asynchronously. @return: A C{Deferred} resulting in the URL content. )rr_)rLkwargsr r r fetch_asyncsracKsVg}|D]}t|fi|}|r||||r|||||qt|dddS)a9 Retrieve a list of URLs asynchronously. @param callback: Optionally, a function that will be fired one time for each successful URL, and will be passed its content and the URL itself. @param errback: Optionally, a function that will be fired one time for each failing URL, and will be passed the failure and the URL itself. @return: A C{DeferredList} whose callback chain will be fired as soon as all downloads have terminated. If an error occurs, the errback chain of the C{DeferredList} will be fired immediatly. T)fireOnOneErrback consumeErrors)ra addCallback addErrbackappendr)urlscallbackerrbackr`resultsrPresultr r r fetch_many_asyncs    rlcCs.|ddd}|durtj||}|S)zReturn the last component of the given C{url}. @param url: The URL to get the filename from. @param directory: Optionally a path to prepend to the returned filename. @note: Any trailing slash in the C{url} will be removed /N)rstripsplitospathjoin)rP directoryfilenamer r r url_to_filenamesrvc s.fdd}fdd}t|f||d|S)aJ Retrieve a list of URLs and save their content as files in a directory. @param urls: The list URLs to fetch. @param directory: The directory to save the files to, the name of the file will equal the last fragment of the URL. @param logger: Optional function to be used to log errors for failed URLs. cs,t|d}t|d}|||dS)Nrtwb)rvopenrGclose)rRrPrufdrwr r rGs    zfetch_to_files..writecsr d|t|jf|S)Nz Couldn't fetch file from %s (%s))r3value)failurerP)loggerr r log_errors  z!fetch_to_files..log_error)rhri)rl)rgrtr~r`rGrr )rtr~r fetch_to_filess rcCsZt}|jddd|jddd|d||\}\}tt||j|j|jddS) Nz--post store_true)actionz--datar")defaultz--cainfo)rQrRrT)r add_option parse_argsprintr_rQrRrT)rLparseroptionsrPr r r tests  r__main__r')NNr)rqsysr.optparsertwisted.internet.deferrtwisted.internet.threadsrtwisted.python.compatrr Exceptionrr rr_rarlrvrrrargvr r r r s,     R