U ɀ^8@sddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z ddl mZmZzddlmZmZWn YnXzddlmZWnddlmZYnXdd lmZdd lTdd lTd d Zed dddZddZddZddZdEddZeddddZ eddddZ!eddd d!Z"d"d#Z#dFd$d%Z$d&d'Z%ed dd(d)Z&ed dd*d+Z'eddd,d-Z(dGd/d0Z)d1Z*ee*Z+e,d2e+d3e+d4Z-ed dd5d6Z.ed ddHd8d9Z/ed:dd;d<Z0dId=d>Z1d?d@Z2dAdBZ3dCdDZ4dS)JN)fnmatch)escape)urljoinurlsplit urlunsplit quote_plus) make_responseResponse)UnionTuple) lru_cache)CONSTS)*cCstjtj|S)N)ospathjoinr ZMIRROR_ROOT)filenamer!/var/www/zmirror/zmirror/utils.py zmirror_rootsr)maxsizecCs |ddS)zM equivalent to s.replace("/",r"\/") :type s: str :rtype: str /\/)replace)srrrs_escsrcCs|ddd}|d}t|ddko:|ddk}t|dksXt|dkr`|r`td fS|rd|d d d|d d fSd|dd d|d dfSd S) u 提取出一个域名的根域名 支持二级顶级域名, 允许包含端口(端口会被舍去) :param domain: eg: dwn.cdn.google.co.jp[:233] :type domain: str :return: root_domain, sub_domain :rtype: Tuple[str, str] 0123456789:.)comnetorgcoedumilgovacN)rstripstripsplitlen target_domainr)domaintempZ is_level2_tldrrrextract_root_domain)s  $r7cCstd|d|d|d|d|d|ddd|ddtd|td|td|d |d |d|dd d|dd d|dd d|dd d|dd d|dd td|td|td|td|td|ttd|ttd|ttd|d S)uN生成各种形式的scheme变体 :type _domain: str :rtype: bool z//zhttp://zhttps://z"%s"z'%s'rz\x2fz\x2Fz\"%s\"z\'%s\'z\\/z\\\/)slashhttphttpsZ double_quoted single_quotedZ hex_lowerZ hex_upperZ slash_escZhttp_escZ https_escZdouble_quoted_escZsingle_quoted_escZslash_double_escZhttp_double_escZhttps_double_escZslash_triple_escZhttp_triple_escZhttps_triple_escZslash_ueZhttp_ueZhttps_ueZdouble_quoted_ueZsingle_quoted_ueZ slash_esc_ueZ http_esc_ueZ https_esc_ue)dictrrr)_domainrrrcalc_domain_replace_prefixCs8        r>cCsddl}|jjS)zeReturns the current line number in our program. :return: current line number :rtype: int rN)inspect currentframef_backf_lineno)r?rrrcurrent_line_numbermsrCWe Got An Unknown ErrorcCs t||S)zO :type errormsg: bytes :type error_code: int :rtype: Response )r)errormsg error_coderrrgenerate_simple_resp_pagevsrHcCs$|}tD]}||kr dSq dS)z Determine whether an mime is text (eg: text/html: True, image/png: False) :param input_mime: str :return: bool TF)lowertext_like_mime_keywords)Z input_mimeZ input_mime_lZ text_wordrrris_mime_represents_texts rLcCs.|d}|dkr|S|d|SdS)ue从content-type中提取出mime, 如 'text/html; encoding=utf-8' --> 'text/html' :rtype: str ;r"N)findrJ) _content_typecrrrextract_mime_from_content_types rQcCst|}|tkr|SdSdS)u0根据content-type确定该资源是否使用CDNFN)rQmime_to_use_cdn)rO_mimerrris_content_type_using_cdnsrTcCs2z||}WnYdSX|dk r*|SdSdS)z0return a blank string if the match group is Noner.N)group)name match_objobjrrr get_groupsrYcCsdS)uC旧版本遗留函数, 已经不再需要, 永远返回空字符串r.r)Z ext_domainZ force_httpsrrr"get_ext_domain_inurl_scheme_prefixsrZcGs,d}|D]}|t|d7}q|d|S)z :return: str r. )strr0)argsoutputargrrrstrxs  r`cCs,|dks tsdS|}t|kr$dSdSdS)u(该user-agent是否满足全局白名单NFT)global_ua_white_namerJ)ua_strrrrcheck_global_ua_passs  rccCstD]}t||rdSqdS)u 域名是否匹配 `domains_whitelist_auto_add_glob_list` 中设置的通配符 :type domain: str :rtype: bool TF)$domains_whitelist_auto_add_glob_listr)r5Z domain_globrrris_domain_match_glob_whitelists recCstD]}||krdSqdS)u# 根据content-type判断是否应该用stream模式传输(服务器下载的同时发送给用户) 视频/音频/图片等二进制内容默认用stream模式传输 :param mime: mime or content-type, eg: "plain/text; encoding=utf-8" :type mime: str :rtype: bool TF)steamed_mime_keywords)mimeZstreamed_keywordrrris_mime_streameds rhr.c CsBd|t|t||dt|t|t|f}|d}t|dS)u生成一个HTML重定向页面 某些浏览器在301/302页面不接受cookies, 所以需要用html重定向页面来传cookie :type target_url: str :type msg: str :type delay_sec: int :rtype: Response u 重定向 (Page Redirect) 
%s
You are now redirecting to %s, if it didn't redirect automatically, please click that link. r utf-8)response) html_escapeencoder )Z target_urlmsgZ delay_secZ resp_contentrrrgenerate_html_redirect_pages rnZzm26_z+(?Pz?)_\.(?P[a-zA-Z0-9-_]+=*)\._z_\.[a-zA-Z\d]+\bcCsdtd|ddkrdSt|}td|}|d|d}t|}td|}|rft|}|j dd }t |d |}|S) u4 将 embed_real_url_to_embedded_url() 编码后的url转换为原来的带有参数的url `cdn_redirect_encode_query_str_into_url`设置依赖于本函数, 详细说明请看配置文件中这个参数的部分 eg: https://cdn.domain.com/a.php_zm24_.cT1zb21ldGhpbmc=._zm24_.css ---> https://foo.com/a.php?q=something (assume it returns an css) (base64 only) eg2: https://cdn.domain.com/a/b/_zm24_.bG92ZT1saXZl._zm24_.jpg ---> https://foo.com/a/b/?love=live (assume it returns an jpg) (base64 only) eg3: https://cdn.domain.com/a/b/_zm24z_.[some long long base64 encoded string]._zm24_.jpg ---> https://foo.com/a/b/?love=live[and a long long query string] (assume it returns an jpg) (gzip + base64) eg4:https://cdn.domain.com/a (no change) ---> (no query string): https://foo.com/a (assume it returns an png) (no change) :param embedded_url: 可能被编码的URL :return: 如果传入的是编码后的URL, 则返回解码后的URL, 否则返回None :type embedded_url: str :rtype: Union[str, None] .__.iNb64rgzipri)encoding?) cdn_url_query_encode_salt®ex_extract_base64_from_embedded_urlsearchrYspanbase64urlsafe_b64decodezlib decompressdecoder)Z embedded_urlmrrZreal_request_url_no_queryZquery_string_byteZ is_gzipped query_stringresultrrr"extract_real_url_from_embedded_urls      rFc Cs|r|dd}n|}t|}|js(|S|j}t|dkrNd}t|}nd}t| }|j dt |d|dt dt |}t |j|j|ddf} |rt| } | S) u 将url的参数(?q=some&foo=bar)编码到url路径中, 并在url末添加一个文件扩展名 在某些对url参数支持不好的CDN中, 可以减少错误 `cdn_redirect_encode_query_str_into_url`设置依赖于本函数, 详细说明可以看配置文件中的对应部分 解码由 extract_real_url_from_embedded_url() 函数进行, 对应的例子也请看这个函数 :rtype: str rrrIzr.rorqrp)rrqueryrlr3r|compressrzurlsafe_b64encoder~r _url_saltrRrschemenetlocr) Z real_url_rawurl_mime escape_slashreal_urlZurl_spZ byte_queryZ gzip_labelZ b64_queryZ mixed_pathrrrrembed_real_url_to_embedded_url7s6    r@cCs:d|kr dSd|kr&d|kr dSdSnd|kr2dSdSd S) uL根据 slash(/) 的格式, 猜测最有可能与之搭配的 colon(:) 格式%r z%25Fz%253Az%253az%3Az%3aNr)r8rrrguess_colon_from_slash]srcCs|rind}t|D]x}|ddkr|dddkrtt||}|rlt|}||krl|d|d|}|rz|||<q|t|d|d7}q|S) Nr.rror$__z....(total:{})r  )dirr\getattrr3formatr`)varto_dictZmax_lenr^rVvaluelengthrrr attributesps   rcCs6|dkrt||S|dkr$t||Std|dS)uT 将文本内容注入到html中 详见 default_config.py 的 `Custom Content Injection` 部分 :param position: 插入位置 :type position: str :param html: 原始html :type html: str :param content: 等待插入的自定义文本内容 :type content: str :return: 处理后的html :rtype: str head_first head_lastzUnknown Injection Position: {}N)inject_content_head_firstinject_content_head_last ValueErrorr)positionhtmlcontentrrrinject_contents   rcCsl|d}|d}|dkr |S|dkrL||krL|d||||dS|d||||dSdS)u 将文本内容插入到head中第一个现有