U I9b@sn ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddl m Z mZmZddlmZddlmZmZddlmZmZmZmZddlZddlZddlmZmZmZm Z m!Z!dd l"m#Z#dd l$m%Z%ej&j'(e%zdd l)m*Z*m+Z+m,Z,m-Z-Wn YnXzdd l.m/Z0Wnd Z1YnXdZ1ej23e4e#j5krze6e#j5ddl7Tdej8krdZ9nd Z9zddl:m;Z;Wn ddldddl?m@Z@e9se>djAe#jBe#jCde>djAe#jDdz ddlETWneFdYnXz ddlGTWneFdYn&XeHIdJddJddZHe>deHe9rddlKZKeKLeKMd eKLeKMd!ddlNTdd"lOmPZPdd#l"mQZQeRrzdd$lSmTZTmUZUeTZVWn e WeFd%d ZRYnXe9se>d&eXeYeZr*e[Z\e]dk rRe[d'e^e]7Z[ee[Z_ne[Z_t`dkrdga`ead(d)t`Da`ebt`pgactcadtdeeHt`D]ZftdeedefjgqebZheheeHeaeiZieireiD]ZftdeefeheefqeijeHneHgZiekJd*d+Zleke[ZmemJd*d+ZneoepZqd,d-d.d/d0d1d2d3d4d5d6d7d8d9d:hZrerseteueHdZveueHdZwexs|d;ZyezsiZ{d;Z|d;Z}gZ~ee~ereae~Z~exsd ZeseZesebZn"eD]Zetckre=dqeeZeD]Zeed7Zq6n d Zd Zd Zesheshd Ze@Zee eaePd?Zdd@dAgedB<ePdCZdeeH<ePdDZdEedF<dGZdHZdIZe]dk rdJee\eee^e]dKee\dLZn ee[ZejdMejdNZejdOejdNZedPZejdQdRdSjAedTdUedVedWjAedXdVdYjAedXdZejdNZeeZd[d\Zeaed]ZepiZeD]L\ZZesĐqeD]2Zed^dkrqejed^ejdNed^<qȐqee9s end_e^eJd`ddddaZdbdcZdddeZddfdgZddhdiZdjdkZdldmZdndoZddpdqZddrdsZeZddtduZeZdvdwZdxdyZdd{d|ZdddZdddZddZe;ddddZdddZŐdddZƐdddZddZe;ddddZe;ddddZddZddZ̐dddZe;ddddZddZddZАdddZddZddZddZddZddZddZddZddZddZڐdddZېdddZddZddZddZߐdddÄZddńZddDŽZddɄZdd˄Zdd̈́ZddτZejddфZedҡddԄZejdddgd׍ddلZejd*dddddddgd׍ejddddddddgd׍dddZdddZedddZedddZiatdD]Zfeeftef< qe re˃ZnebZz*e9 r ddlKZKeKLeKMdddlTWn YnXe rfzddlmZWne=dYnXe rzddlmZWnd Ze=dYnXe rzddlmZmZWnd ZeFdYnXe rPeD]ZZz"eeded<eeddWn0ek r.ZzeFdeW5dZ[XYnX qe jeddZeedk rjeFdedS( N)fnmatch)timesleep process_time)escape)datetime timedelta)urljoinurlsplit urlunsplit quote_plus)Flaskrequest make_responseResponseredirect)CONSTS)InsecureRequestWarning)UnionListAnyTuple)detectFT)*ZZMIRROR_UNITTEST) lru_cachezpackage fastcache not found, fallback to stdlib lru_cache, no FUNCTION is effected, only maybe a bit slower. Considering install it using "pip3 install fastcache"z,lru_cache loaded successfully from fastcache)ZmirrorThreadLocalz+zmirror version: {version} author: {author})versionauthorzGithub: {site_url})Zsite_urluthe config_default.py is missing, this program may not works normally config_default.py 文件丢失, 这会导致配置文件不向后兼容, 请重新下载一份 config_default.pyuthe config_default.py is missing, fallback to default configs(if we can), please COPY the config_default.py to config.py, and change it's content, or use the configs in the more_configs folder 自定义配置文件 config.py 丢失或存在错误, 将使用默认设置, 请将 config_default.py 复制一份为 config.py, 并根据自己的需求修改里面的设置(或者使用 more_configs 中的配置文件)./ https://http://zconfig file found, mirroring: z zmirror.utilszzmirror.connection_pool)LRUDict)connection_pool) FileCacheget_expire_from_mimezLCan Not Create Local File Cache, local file cache is disabled automatically.zLocal file cache enabled:cCs&g|]}|dddddqS)rr r!r")stripreplace).0dr,#/var/www/zmirror/zmirror/zmirror.py sr./\/ content-typedateexpiresz cache-controlz last-modifiedZserverlocationz accept-rangesaccess-control-allow-originzaccess-control-allow-headerszaccess-control-allow-methodszaccess-control-expose-headerszaccess-control-max-agez access-control-allow-credentialstiming-allow-originr,zAn isolated domain:zOwould not have effect because it did not appears in the `external_domains` list)strictiz image/pngiBz&www.fake-domain.com/folder/foo/bar.pngdiztarget.domain.com)z example.comz/path/no/query/stringz(?::|%(?:25)?3[Aa])z3(?:\\*(?:/|x2[Ff])|%(?:(?:25)?5[Cc]%)*(?:25)?2[Ff])z2(?:\\*["']|%(?:(?:25)?5[Cc]%)*2(?:52)?[27]|")(?:|)a9(?P\b(?:(?:src|href|action)\s*=|url\s*\(|url\s*:|@import\s*|"\s*:)\s*)(?P["'])?(?P(?P(?:https?:)?\\?/\\?/)(?P(?:[-a-z0-9]+\.)+[a-z]+(?P:\d{1,5})?))?(?P[^\s;:\\+$?#'"\{}]*?(?P\?[^\s?#'"]*?)?)(?P["')])(?P\W)flagsz\bdomain=(\.?([\w-]+\.)+\w+)\bz*(?P[pP]ath)=(?P[\w\._/-]+?;)z(?P (?P#(?:https?(?P{REGEX_COLON}))? REGEX_COLON%(?P%s)(?P=scheme_slash)z)?z<(?P(?(scheme_slash)(?P=scheme_slash)|{REGEX_SLASH}))) REGEX_SLASHzCextdomains(?(slash2)(?P=slash2)|{REGEX_SLASH})(?Phttps-)?z#(?P(?:[\w-]+\.)+\w+)\bcsddlm}|ddtDttfdddd}d d |d }td d d j t ddt d d dj t dd d|dj t ddS)u产生 regex_basic_mirrorlization 用一个函数包裹起来是因为在 try_match_and_add_domain_to_rewrite_white_list() 中需要动态修改 external_domains, 修改以后可能需要随之生成新的正则, 包裹一下比较容易调用 r)Countercss"|]}t|ddVqdS).N)rersplit)r*xr,r,r- Usz8_regex_generate__basic_mirrorlization..cs|SNr,rIcr,r-Vz7_regex_generate__basic_mirrorlization..T)keyreverser9r:r;r>r?r@rBz(?P{REGEX_QUOTE})) REGEX_QUOTEz&(?P([a-zA-Z0-9-]+\.){1,5}%s)\bz=(?P(?(scheme_slash)(?P=scheme_slash)|{SLASH}))?)ZSLASHz(?(quote)(?P=quote))) collectionsrDallowed_domains_setsortedlistkeysjoinrGcompileformatrArCrS)rDZregex_all_remote_tldr,rMr-%_regex_generate__basic_mirrorlizationMs6    r\z!zmirror_verify=[a-zA-Z0-9]+\b;? ? url_regexZunittestrE)Z static_folderZtemplate_foldercCsdd}t||S)u response_text_basic_rewrite() 的实验性升级版本, 默认启用 *v0.28.1.dev* 之前版本是在正则中匹配所有允许的域名, 现在改为匹配所有可能允许的TLD, 可以带来一些性能的提升, 并且容易进行动态域名添加和通配符支持 *v0.28.2* 进一步优化正则, 性能提升 47% 左右 (速度约为传统暴力替换的4.4倍) *v0.28.3* 目前来看该功能工作得相当好, 由实验性特性改为正式使用 移除旧版 response_text_basic_rewrite(), 只保留一个为了向下兼容的 alias :param text: 远程响应文本 :type text: str :return: 重写后的响应文本 :rtype: str cSstd|}|tkr&trt|s&|Std|}td|p@|p@d}td|pRt|}trdtd|nt}|r|t kr||d|||}n||}td|}|r|||Std|rt d|d||S|d |SdS) Ndomain suffix_slash scheme_slashr/colonr'Z extdomainsquote) get_grouprU"enable_automatic_domains_whitelist.try_match_and_add_domain_to_rewrite_white_listgroupZguess_colon_from_slash my_host_port my_host_namer)domain_alias_to_target_setmy_host_scheme)m remote_domainr_ZslashraZ _my_host_nameZcorerbr,r,r-regex_reassembles&      z.regex_reassemble)regex_basic_mirrorlizationsub)textrnr,r,r-"response_text_basic_mirrorlizations!rrcCsTtdk r tStr@tD]*}z|j|dWn YqX|SqtrPt|dSdS)u 试图解析并返回二进制串的编码, 如果失败, 则返回 None :param byte_content: 待解码的二进制串 :type byte_content: bytes :return: 编码类型或None :rtype: Union[str, None] Nencodingrt)Z force_decode_remote_using_encodeZpossible_charsetsdecodecchardet_available c_chardet)Z byte_contentZcharsetr,r,r-encoding_detects   rxcCstrtj|dtr tj|d|rzlttt t t t t ttttttWntdtYnXdS)u  清理程序运行中产生的垃圾, 在程序运行期间会被自动定期调用 包括各种重写缓存, 文件缓存等 默认仅清理过期的 :param is_force_flush: 是否无视有效期, 清理所有缓存 :type is_force_flush: bool )Z force_flush)Zforce_flush_allZ!ErrorWhenCleaningFunctionLruCacheN)enable_connection_keep_aliver$clearlocal_cache_enablecacheZcheck_all_expireurl_to_use_cdnis_domain_match_glob_whitelist cache_clearis_mime_streamed"extract_real_url_from_embedded_urlembed_real_url_to_embedded_urlcheck_global_ua_passis_mime_represents_textextract_mime_from_content_typeis_content_type_using_cdnis_ua_in_whitelistverify_ip_hash_cookieis_denied_because_of_spideris_ip_not_in_allow_rangeerrprint traceback print_exc)Zis_force_flushr,r,r- cache_cleans,   rcCs|szftd|dt|ddt|d|d}|dkrNtdt|||dd|d iWntd |tYnXtst t krt ndSt |d d |d dt|fdS)u 定时任务容器. 调用目标函数, 并在运行结束后创建下一次定时 :param task_dict: 定时任务的相关参数, dict { "target":目标函数(可调用的函数对象,不是函数名字符串) 必须, "iterval":任务延时(秒) 可选, "priority":优先级 可选, "name":定时任务别名 可选 "args":位置型参数 (arg1,arg2) 可选, "kwargs":键值型参数 {key:value,} 可选, } :param add_task_only: 是否只添加定时任务而不执行 z CronTask:nametargetzTarget:Nztarget is not given in argsr,kwargsZErrorWhenProcessingCronTasksZinterval,Zpriorityi) infoprintgetstr ValueErrorrrrenable_cron_tasks threadingcurrent_thread main_threadexittask_schedulerZentercron_task_container)Z task_dict add_task_onlyZ target_funcr,r,r-rs.&      rcCsXts ttkrtndStdz tWqtdt YqXqdS)uR定时任务宿主, 每分钟检查一次列表, 运行时间到了的定时任务N<ZErrorDuringExecutingCronTasks) rrrrrrrrunrrrr,r,r,r-cron_task_host5s rcCsFtjdkrgt_n ttjt_tj||ftd|d|dtjdS)u 添加临时域名替换列表 用于纯文本域名替换, 见 `plain_replace_domain_alias` 选项 :param source_domain: 被替换的域名 :param replaced_to_domain: 替换成这个域名 :type source_domain: str :type replaced_to_domain: str NzA domaintozadded to temporary_domain_alias)parsetemporary_domain_aliasrWappenddbgprint)Z source_domainZreplaced_to_domainr,r,r-add_temporary_domain_aliasGs   rcCs|tkS)u是否是外部域名)domains_alias_to_target_domainr^r,r,r-is_external_domainZsrc Cs|dks |sdS|tkrdS|s,t|s,dStd|dtt}||t|at|t|t |t |<t a z0t tdddd }||d W5QRXWntYnXdS) u 若域名与`domains_whitelist_auto_add_glob_list`中的通配符匹配, 则加入 external_domains 列表 被加入 external_domains 列表的域名, 会被应用重写机制 用于在程序运行过程中动态添加域名到external_domains中 也可在外部函数(custom_func.py)中使用 关于 external_domains 更详细的说明, 请看 default_config.py 中对应的文档 :type domain: str :type force_add: bool :rtype: bool NFTz A domain:z"was added to external_domains listzautomatic_domains_whitelist.logautf-8rs )rUr~rrWexternal_domainsrtupleexternal_domains_setaddcalc_domain_replace_prefix prefix_buffr\roopen zmirror_rootwriterr)r^Z force_addZ_bufffpr,r,r-rf`s(       rfc Cszd}d}i}|dkrt}n8d|kr4d}|dd}d|krLd}|dd}t|}|ddd kr td |ddd}|j}|jpd|jrd |jnd }|dd dkr|d d}d}nt|}t|}|r|dd}|rt |}||d<||d<||d<t|dj|d<|St|}|r:|dd}|rHt |}t |d<t dk|d<||d<t|dj|d<|S)u 解析镜像url(可能含有extdomains), 并提取出原始url信息 可以不是完整的url, 只需要有 path 部分即可(query_string也可以有) 若参数留空, 则使用当前用户正在请求的url 支持json (处理 \/ 和 \. 的转义) :rtype: dict[str, Union[str, bool]] :return: {'domain':str, 'is_https':bool, 'path':str, 'path_query':str} FNr0Tr/z\.rE /extdomains///?r!zhttps-r^is_https path_querypathr ) extract_url_path_and_queryr)r lstripnetlocrqueryis_target_domain_use_httpsclient_requests_text_rewrites_esc target_domain target_scheme) Z mirror_urlZ_is_escaped_dotZ_is_escaped_slashresultZinput_path_queryrH real_domainZreal_path_query _is_httpsr,r,r-decode_mirror_urlsP       rc Cs|r|dd}n|}t|}d|jddkr4|S|pF|jpFtjpFt}|tkr`td|d|S|dk r|dd d krd t }q|s|j rt }qd }nd }t |rd|}nd }td |d |dt d|t ||dt|d} |jr| d|j7} |rt| } | S)zconvert url from remote to mirror url :type raw_url_or_path: str :type remote_domain: str :type is_scheme: bool :type is_escape: bool :rtype: str zr\/r/rNrzdomain:z is not in allowed_domains_setFrcrr!zraw_url_or_path=z ; domain: z;myurl_prefix:z; middle_part: #)r)r rrrrmrrUrrischeme myurl_prefixrr rrZfragmentr) Zraw_url_or_pathrm is_schemeZ is_escapeZ_raw_url_or_pathspr^Z our_prefixZ middle_partrr,r,r-encode_mirror_urls:     $  rcCs,tdkr dStdkrdS|tkr$dSdSdS)u&请求目标域名时是否使用httpsZNONEFZALLTN)Zforce_https_domainsrr,r,r-rsrcCst|dS)uK添加域名到ssrf白名单, 不支持通配符 :type domain: str N)rUrrr,r,r-add_ssrf_allowed_domainsr error_dumpc Csddl}ztjt|s(tt|td}ddl }tt |t t|ddttddd}|dk r||||d<tjtjt||d}t|d }| |||jW5QRX|WSYdSXdS) u* dump当前状态到文件 :param folder: 文件夹名 :type folder: str :param our_response: Flask返回对象, 可选 :type our_response: Response :param msg: 额外的信息 :type msg: str :return: dump下来的文件绝对路径 :rtype: Union[str, None] rNzsnapshot_%Y-%m-%d_%H-%M-%ST)Zto_dict)rrmsgrconfigZ FlaskRequestZ OurResponsez.dumpwb)pickleosrexistsrmkdirrnowstrftimerrdumpr format_excZ attributesrZfreezeabspathrYrZHIGHEST_PROTOCOL) ZfolderrZ our_responserZ _time_strrZsnapshotdump_file_pathrr,r,r-dump_zmirror_snapshots,    r Unknown Errorc Cs|rtt|t|tr&|}t|d}d}tddtt D]$}|dj |t t t |d7}qFdj |||rt tnd|tjtjd }|st||S|Sd S) z :type content_only: bool :type errormsg: Union(str, bytes) :type error_code: int :type is_traceback: bool :rtype: Union[Response, str] rr!cSs|ddko|dddkS)Nr___r,rLr,r,r-rOTrPz%generate_error_page..z*{attrib}{value})attribvalueu: zmirror internal error

zmirror internal error

An fatal error occurs. 服务器中运行的zmirror出现一个内部错误.

If you are visitor 如果你是访客

This site is temporary unavailable because some internal error
Please contact your site admin.
该镜像站暂时出现了临时的内部故障, 请联系网站管理员

If you are admin

You can find full detail log in your server's log.
For apache, typically at /var/log/apache2/YOUR_SITE_NAME_error.log
tips: you can use tail -n 100 -f YOUR_SITE_NAME_error.log to view real-time log

If you can't solve it by your self, here are some ways may help:

Snapshot Dump

An snapshot has been dumped to {dump_file_path}
You can load it using (Python3 code) pickle.load(open(r"{dump_file_path}","rb"))
The snapshot contains information which may be helpful for debug

Detail

{request_detail}
AttribValue

Additional Information

{errormsg}

Traceback

{traceback_str}
Powered by zmirror {version}
{official_site}
zNone or not displayed)errormsgrequest_detailZ traceback_strrrZ official_siteN)rrr isinstancebytesrurfilterdirrr[ html_escaper__getattribute__rr __VERSION____GITHUB_URL__rencode)r error_code is_tracebackZ content_onlyrrrZ error_pager,r,r-generate_error_pageAs.    *0rcCst|dd}|jdd|S)z:rtype Responsei0) content_typestatuszX-Cachez FileHit-304)rheadersr)Z _content_typerr,r,r-generate_304_responses rcCst}|D]$}||||ttdd7}qtt|jdddd}t|dkrb|d7}qLtt|tjdddd}t|dkr|d7}q||S) u 生成一个标示用户身份的hash 在 human_ip_verification 功能中使用 hash一共14位 hash(前7位+salt) = 后7位 以此来进行验证 :rtype str ri@TrrsrcN0) &human_ip_verification_answers_hash_strrrandomZrandinthexzlibadler32rlen)Z input_dictZstrbuffrQinput_key_hash output_hashr,r,r-generate_ip_verify_hashs"  "  r maxsizecCsfzR|dd}|dd}tt|tjdddd}||krJWdSWdSWnYdSXdS)u 根据cookie中的hash判断是否允许用户访问 在 human_ip_verification 功能中使用 hash一共14位 hash(前7位+salt) = 后7位 以此来进行验证 :type hash_cookie_value: str :rtype: bool NrrsrcTF)rrrrr)Zhash_cookie_valuerrZcalculated_hashr,r,r-rs    rGETcCstr|dkrt|rt|}t|}||d|d<tdkr^td||ddt|tj ||t|t t j | d|d dS) uQ更新 local_cache 中缓存的资源, 追加content 在stream模式中使用rFwithout_contentZLocalCache_UpdateCacheN last_modified)obj_sizer3r info_dict)r{r| is_cachedget_infoget_objset_data verbose_levelrrput_objr&rmimer)urlcontentmethodrrespr,r,r-update_content_in_local_caches   r c Cstjdks|jdkrdStd|d||r@t|}d|_d}n|}ttjj}tjj dd}t j ||t tj||||dd dS) a; put our response object(headers included) to local cache :param without_content: for stream mode use :param url: client request url :param _our_resp: our response(flask response object) to client, would be storge :type url: str :type _our_resp: Response :type without_content: bool rNz PuttingCache:zwithout_content:rz Last-Modified)rr)r3rrr)rr status_codercopyresponserremote_responserrrr|rr&r)rZ _our_resprZour_resprrr,r,r-put_response_to_local_caches(   r&cCstrtjdkrt|r|dk rLd|krLt||ddrLtd|tSt |}|ddrfdSt |}t |t s~t tdd|SndSdS) u 尝试从本地缓存中取出响应 :param url: real url with query string :type client_header: dict :rtype: Union[Response, None] rNzif-modified-sincezFileCacheHit-304rTzx-zmirror-cacheZFileHit)r{rrr|rZ is_unchangedrrrrrrrAssertionErrorset_extra_resp_header)r client_headerZ cached_inforr,r,r-try_get_cached_responses     r*cCsLtd|}td|}td|}td|}td|}td|}|}d|ksTd|krfd}|dd }nd }|rd |krd |kr|r|d ksd|krd |ks|r||ks|sd|krdtjkr|ddd ks|sd|kr|Strt||ptj} | tkr|St tj |}t j dkr2d|kr2|dd }| d sFd |}dtjkrht | d |d } n| d |d } | tkrd| }tr| tkrt| d} nd } | rttt|t} n$|sd} nd|krdt} nt} t | |} | r tr t| t| d|d} |r.t| } ||| |td|}|S)z Reassemble url parts split by the regex. :param match_obj: match object of stdlib re :return: re assembled url string (included prefix(url= etc..) and suffix.) :rtype: str prefix quote_left quote_rightrr^rr0Tr/Frimportr;r'zsrc=Z javascriptNr")z/../rrr!httpr)url_mimeZ escape_slashZ right_suffix)rdrgr)rrrerfrmrUr remote_pathsys version_info startswithrrenable_static_resource_CDNr}rk CDN_domainsrrrcdn_domains_numberrir&cdn_redirect_encode_query_str_into_urlrr) match_objr+r,r-rZ match_domainrZwhole_match_stringZrequire_slash_escaper^ url_no_schemeZ_this_url_mime_cdnZreplace_to_scheme_domainZreassembled_urlZ reassembledr,r,r-regex_url_reassembles                     r>cCs0|}t|krdStD]}||krdSqdS)u 当机器人或蜘蛛的请求被ban时, 检查它是否处在允许的白名单内 被 is_denied_because_of_spider() 调用 :type ua_str: str TF)lowerZglobal_ua_white_nameZspider_ua_white_list)ua_strZ allowed_uar,r,r-rsrcCsD|}d|ksd|krtjtkrt|ttjd <tr|sttj|tjjjd d Stdkr<|t|7}td|d|dt|dq)rrSZQueueZ/stream_transfer_async_preload_max_packages_sizerThreadrVrr%startrZEmpty warnprintrrZ task_doner{rr=r}r  remote_urlrrrr)Z total_size _start_timeZ_content_bufferZ_disable_cache_temporaryrTtrUr,r,r-iter_streamed_response_async sJ    r_cCs|rdtjd<t}nt\}tjd<tdtjjt|tjjd}tjjD]^}| }|t kr|dkrtjj|}td|t rt |dtj }t|dd |j|<td |j|n|d krttjrd tjkrtjd |j|<ntjj||j|<n|dkrrtdkrt|j|<nVtdkrPtjdpBtjdpBt}||j|<n |dkrft|jd<n t|j|<ntjj||j|<|dkrLtD]}|jdt|qqLtd|j|S)z Copy and parse remote server's response headers, generate our flask response object :type is_streamed: bool :return: flask response object :rtype: Response r req_time_bodyZRemoteRespHeaders)rr4z 302 locationzmwm/headers-locationFTz302 rewrite locationr1rz; charset=utf-8)r5r6Nz_*_originZOriginr5zAccess-Control-Allow-Origin set-cookie Set-CookiezOurRespHeaders: )rrr_response_content_rewriterr%rrr"r@allowed_remote_response_headerscustom_text_rewriter_enablecustom_response_text_rewriterr\rrrrZcustom_allowed_originrrrresponse_cookies_deep_copyrresponse_cookie_rewrite) is_streamedrrZ header_keyZheader_key_lowerZ _locationZ_origin cookie_stringr,r,r- copy_response=sF                rlcCstjjjjj}g}|D]\}}|dkrtdkrV|dd}|dd}|dd}d|krt rtt d |}n(t d k rtj t krt d tj d |}||q|S) a It's a BAD hack to get RAW cookies headers, but so far, we don't have better way. We'd go DEEP inside the urllib's private method to get raw headers raw_headers example: [('Cache-Control', 'private'), ('Content-Length', '48234'), ('Content-Type', 'text/html; Charset=utf-8'), ('Server', 'Microsoft-IIS/8.5'), ('Set-Cookie','BoardList=BoardID=Show; expires=Mon, 02-May-2016 16:00:00 GMT; path=/'), ('Set-Cookie','aspsky=abcefgh; expires=Sun, 24-Apr-2016 16:00:00 GMT; path=/; HttpOnly'), ('Set-Cookie', 'ASPSESSIONIDSCSSDSSQ=OGKMLAHDHBFDJCDMGBOAGOMJ; path=/'), ('X-Powered-By', 'ASP.NET'), ('Date', 'Tue, 26 Apr 2016 12:32:40 GMT')] rbr"zSecure;r!z;Secure;z; SecureZhttponlyzpath=/;Nz\g=/extdomains/z\g)rr%rawZ_original_responserZ_headersr@rkr)Z&enable_aggressive_cookies_path_rewriteregex_cookie_path_rewriterrprmrjr)Z raw_headersZheader_cookies_string_listrrr,r,r-rhs&         rhc Cst}tjj}t|}ttjs6tdtj||fStdkrbtdtjtjj dd|ddt tjj}|dk r~|tj_ tjj }t dk rt |krt dttr(t|tjtj}t|tr|}n@t|tst|tr|\}}|rt dtj|jdd |fSt dk r(t |kr(t d tt|}t dk rPt |krPt d ttrtjd krtD]L\}}|D]<} | d } | dk r| tjsqvt||| d}qvqj|jdd |fS)zw Rewrite requests response's content's url. Auto skip binary (based on MIME). :return: Tuple[bytes, float] ZBinaryr0z Text-likeNrXzDStringTrace: appears in the RAW remote response text, code line no. ZSkip_builtin_rewriterrsz>StringTrace: appears after custom text rewrite, code line no. z:StringTrace: appears after builtin rewrite, code line no. z text/htmlr]r) rrr%rrrrrrrqrxrtdeveloper_string_tracercurrent_line_numberrfrgr\rrrrWrrrresponse_text_rewritecustom_inject_contentitemsrmatchr=Zinject_content) r]Z_contentr`rt resp_textZ resp_text2Zis_skip_builtin_rewriteZpositionrtitemrr,r,r-rdsN           rdcOs ddlm}|dtt||S)u本函数在v0.28.3被移除, 对本函数的调用会被映射出去 如果需要查看本函数代码, 请查看git历史到 v0.28.3 以前 r)warnz[This function is deprecated since v0.28.3, use response_text_basic_mirrorlization() instead)warningsrxDeprecationWarningrr)rrrxr,r,r-response_text_basic_rewrites  r{cCsHtr$ttjD]\}}|||}q|ddkrrprqrrtarget_domain_rootmy_host_name_no_port)rvZbefore_replaceZ after_replacer,r,r-rrs*      rrcCstdt|}|S)z` rewrite response cookie string's domain to `my_host_name` :type cookie_string: str r})regex_cookie_rewriterrpr)rkr,r,r-ri+sricCs:tjr&tjrdnd}t|tjtjSttttjSdS)uU 组装目标服务器URL, 即生成 parse.remote_url 的值 :rtype: str r r"N)rrrr rmremote_path_queryrr)rr,r,r-assemble_remote_url8srcCs.tjtkr*ttjs*tr&ttjdSdSdS)u SSRF防护, 第一层, 在请求刚开始时被调用, 检查域名是否允许 :return: 如果请求触发了SSRF防护, 则返回True :rtype: bool FT)rrmrUrf+developer_temporary_disable_ssrf_preventionrr,r,r,r-ssrf_check_layer_1Fs   rcCsi}tdtjtjD]\}}|}|dkr2qq|dkrF|dkrFqq|dkrd|ks^d|krd}d|krr|d 7}d |kr|d 7}|r|||<qqt|||<|d krtd||||<qtd ||S) u Extract necessary client header, filter out some. 对于浏览器请求头的策略是黑名单制, 在黑名单中的头会被剔除, 其余所有请求头都会被保留 对于浏览器请求头, zmirror会移除掉其中的 host和content-length 并重写其中的cookie头, 把里面可能存在的本站域名修改为远程服务器的域名 :return: 重写后的请求头 :rtype: dict zBrowserRequestHeaders:)hostzcontent-lengthr1r!zaccept-encodingZbrZsdchZgzipzgzip, ZdeflateZcookiezFilteredBrowserRequestHeaders:)rrrr@r$regex_remove__zmirror_verify__headerrp)Zrewrited_headersZ head_nameZ head_valueZ head_name_lZ _str_buffr,r,r-extract_client_headerWs2     rcCsZdd}t||}tdk r0t|kr0tdttt|}|tt}t d|d||S)a Rewrite proxy domain to origin domain, extdomains supported. Also Support urlencoded url. This usually used in rewriting request params eg. http://foo.bar/extdomains/accounts.google.com to http://accounts.google.com eg2. foo.bar/foobar to www.google.com/foobar eg3. http%3a%2f%2fg.zju.tools%2fextdomains%2Faccounts.google.com%2f233 to http%3a%2f%2faccounts.google.com%2f233 :type raw_text: str :rtype: str cSstd|}|d}td|}ttd|}|d}d}|rxd|krl|sRt|r`|d|7}n |d|7}||d 7}||7}|S) Nrrar`rrr!r2Zhttpsrc)rdrgboolr)r<rrar`rrrr,r,r-replace_to_real_domains       z.replace_to_real_domainNzAStringTrace: appears client_requests_text_rewrite, code line no. zClientRequestedUrl: z<- Has Been Rewrited To ->) !regex_request_rewriter_extdomainsrprprrq"regex_request_rewriter_main_domainrr)rir)Zraw_textrZreplacedr,r,r-rs    rcCs<|dkrtj}t|}|jpd}|s8|jr8|d|j7}|S)z Convert http://foo.bar.com/aaa/p.html?x=y to /aaa/p.html?x=y :param no_query: :type full_url: str :param full_url: full url :return: str Nr/r)rrr rr)Zfull_urlZno_queryrHrr,r,r-rs   rc Cst|j}td|d||tkr.ts.td||s6d}tj|||||d}t r^t |}nt }t tj d<|j|tdtt d}t tj dtj d <td tj d d d td krt|jjd|jd|jtd|jj|rtd|jjtd|j|S)uy实际发送请求到目标服务器, 对于重定向, 原样返回给用户 被request_remote_site_and_parse()调用ZFinalRequestUrlZ FinalHostnamez5Trying to access an OUT-OF-ZONE domain(SSRF Layer 2):N)rZparamsdataZreq_start_timeF)ZproxiesZallow_redirectsstreamZverifyreq_time_headerz RequestTime:rvr0zFinalSentToRemoteRequestUrl:z Rem Resp Stat: zRemoteRequestHeaders: zRemoteRequestRawData: zRemoteResponseHeaders: )r rrrUrConnectionAbortedErrorrequestsZRequestZprepareryr$Z get_sessionZSessionrrsendrequests_proxiesenable_stream_content_transferZdeveloper_do_not_verify_sslrrrrr"rZbody) rrrZ param_getrZfinal_hostnameZ prepped_reqZ_sessionrr,r,r- send_requestsB       rcCst}t|}|dk rBz|j|d}Wnd}Yn Xt|}trxt|tr\|j|d}tj|d|krxt dt ||fS)u 解析出浏览者发送过来的data, 如果是文本, 则进行重写 如果是文本, 则对文本内容进行重写后返回str 如果是二进制则, 则原样返回, 不进行任何处理 (bytes) :rtype: Union[str, bytes, None] NrszFStringTrace: appears after client_requests_bin_rewrite, code line no. ) rget_datarxrurrprrrrrq)rrtr,r,r-prepare_client_request_datas   rcCsttjd}tjddkr0tddtjdtjddk rxtjsxtddtjd td dttjdtd d tjt rtjst d |S)u4 生成我们的响应 :rtype: Response )rjrgh㈵>zX-Header-Req-Time%.4f start_timeNzX-Body-Req-Timer`X-Compute-Timez X-Powered-Byz zmirror/%sZtraffic) rlrstreamed_our_responserr(rrrrZdeveloper_dump_all_trafficsr)rr,r,r-generate_our_response6s  rc Csbtjjddt_ttjt_tr8ttjs8t dddSt oDt tjt_ tjjddt_ dtj kodtj kod tj kod tj kotjjjd kotjjd kt_td krtdtjdtj dtjdtd d tr^tjr^tjtkr^dtjjkrtjjd}ntj r d}n ttjj}dtj|gttj<ttjrRdttjd<tdtjn tdtjdS)u处理远程服务器的响应z Content-Typer!s'This site is just for static resources.)rz Cache-Controlzno-storezmust-revalidatez max-age=0Zprivaterr!rzResponse Content-Type:z IsStreamed:z cacheable:ZLinerzContent-LengthrFFTrzCDN enabled for:zCDN disabled for:N)rr%rrrrrZonly_serve_static_resourcesrgenerate_simple_resp_pagerrrZ cache_controlrrr" cacheablerrrqr8r=r}rr)Zlengthr,r,r-parse_remote_responseNsJ        rrc Cstj}tttj}d}ttd|D]\}}||kr@q,||d<ztt |t j tj tj d}WnYq,YnXd|jkrdkrnn0td|dd||dks,|dkrq,n|\}}nPd |jkrd krnn4||dkr|dkr,||f}q,n|dk r,|\}}nq,td |d |td |ttj|dd}tdt jd ||t _|t|t jf<z>ttdddd"}|dt|t j|W5QRXWn YnXt|jt _t|SdS)u 猜测url所对应的正确域名 当响应码为 404 或 500 时, 很有可能是把请求发送到了错误的域名 而应该被发送到的正确域名, 很有可能在最近几次请求的域名中 本函数会尝试最近使用的域名, 如果其中有出现响应码为 200 的, 那么就认为这条url对应这个域名 相当于发生了一次隐式url重写 * 本函数很可能会改写 parse 与 request :rtype: Union[Tuple[Response, float], None] NrrrriiWzDomain guess failed:rrrizdomain guess successful, fromrzX-Domain-GuessTrmrzShadow rewriting, fromzdomain_guess.logrrrsz{} {} {} --> {} )rrmrWr r\ enumeraterecent_domainsrXrr rrr)request_data_encodedr"rr(rrrdomain_guess_cacherrrrr[rrassemble_parse) ZdepthZcurrent_domainrZ redirectedir^r rewrited_urlZfwr,r,r-guess_correct_domains`       * rcCs@ttjtjtjtjdt_tjjtjkrz )rrrZstrxr cache_inforrrrrrr~rrrrZ active_countrj)outputr,r,r-zmirror_statuss&  rz/ip_ban_verify_pagePOST)methodsc Cs tjdkrtdtjd}ttD]6\}}|d|d|t|dkrNt|dndf7}q"tD].\}}}|d|t rvd ndt|t|f7}q^d tj kr|d ttj d 7}d tt tt t rd ndt |fStjdkrtdtjttD]z\}}tj t|d}|dkr:t r qntd|ddS||dkrbtd|ddSt rq~qt r~tddSi}tD]H\}}}|tjkstj|std|dStj|||<qd} d tjkrFzttj d jdd} Wn(tdttj d ddYSXt| j} | rF| tkrFd} tr`t|s`tddSt| td} trt|} | jd| t !t"t#d t#d!d"d#| |d$<n^t$r t%|t} td%| | d&krtddS| jd| t !t"t#d t#d!d"d#| |d$<t&tj|d'| Sd&S)(u生成一个身份验证页面rz Verifying IP:r!zN%s
rr0rcz'%s %s
u)(必填) razFu] %s

%s

这样的验证只会出现一次，通过后您会被加入白名单，之后相同IP的访问不会再需要验证。
提示: 由于手机和宽带IP经常会发生改变，您可能会多次看到这一页面。

%s 
%s
%s
u7只需要回答出以下任意一个问题即可u+你需要回答出以下所有问题rzVerifying Request FormsPlease answer question: r!rsWrong answer in: s#Please answer at least ONE questionsParam Missing or Blank: r/rrsz#Unable to decode origin from value:Trs!Verification Failed, please checkrr)Zdaysi)r3Zmax_ageZ__zmirror_verifyzSelfGeneratedCookie:Nr)'rrrrrhuman_ip_verification_questionsrrZ%human_ip_verification_identity_recordZ4human_ip_verification_answer_any_one_questions_is_okrrZhuman_ip_verification_titleZ!human_ip_verification_descriptionZformrrrrZurlsafe_b64decoderurr rriidentity_verify_requiredcustom_identity_verifyZgenerate_html_redirect_pageZ!human_ip_verification_success_msgrr Z set_cookierrrZ4human_ip_verification_whitelist_cookies_expires_daysrcustom_generate_access_cookierM) Z form_bodyZq_idZ _questionZrec_explain_stringZrec_nameZ input_typeZsubmitted_answerZ record_dictZ form_typerarr_hashr,r,r-ip_ban_verify_page s  $                     rZOPTIONSZPUTZDELETEZHEADZPATCHz/cCsrzRt|d}tjD]\}}|j||qtjD]\}}|jd|q8WntddYSX|SdS)uN入口函数的壳, 只是包了一层异常处理, 实际是 main_function() ) input_pathrcTrN) main_functionrZextra_resp_headersrtrrBZ extra_cookiesrr)rrrrrkr,r,r- zmirror_enter{ s rcCstdttjt_ttjd<tt}|dk rDtd|St }|dk rV|St }t rlt ddSt t_t}|dk r|St\t_t_ttt}trtjrttj|tjdtd|S) u8本程序的实际入口函数 :rtype: Response z-----BeginRequest-----rNz"-----EndRequest(filtered out)-----s,SSRF Prevention! Your domain is NOT ALLOWED.r)rz-----EndRequest-----)rrZinitrrrrrrrrrrrr)rr request_dataZrequest_data_encodingrrrr{rr&r\r)rrrrr,r,r-r s6   rz/crossdomain.xmlcCs tdddS)NaD ztext/x-cross-domain-policyr)rr,r,r,r-r srz/about_zmirrorcCs"tdjtjtjtjttdddS)Nzzmirror version: {version} Author: {author} Github: {github_url} Note: Love Luciaz Forever! Mirroring: {source_site} This site: {my_domain} )rrZ github_urlZ source_siteZ my_domainz text/plainr)rr[rr __AUTHOR__rrrir,r,r,r- about_zmirror s r custom_func)rgzuCannot import custom_response_text_rewriter custom_func.py, `custom_text_rewriter` is now disabled(if it was enabled))rzoCannot import custom_identity_verify from custom_func.py, `identity_verify` is now disabled (if it was enabled))rrzCannot import custom_generate_access_cookie and custom_generate_access_cookie from custom_func.py, `enable_custom_access_cookie_generate_and_verify` is now disabled (if it was enabled)r)rZUnableToInitCronTask)rrW__main__z#Please use `python3 wsgi.py` to run)F)F)F)N)NNF)rNN)rrFF)N)r)F)N)N)F)NF)rNNN)r)r/)r/(rr5rGr#rZschedrSrrrrNrrrrrZhtmlrrrrZ urllib.parser r r r ZurllibrZflaskr rrrrr!rZ$requests.packages.urllib3.exceptionsrZpackagesZurllib3Zdisable_warningstypingrrrrZcchardetrrwrvrrgetcwdZ ZMIRROR_ROOTchdirZexternal_pkgs.ColorfulPyPrintenvironZ unittest_modeZ fastcacher functoolsr[rZ threadlocalrr[rrrZconfig_defaultrrrr(r)Z importlibZreloadZ import_moduleZutilsZlru_dictr#r$r{Z cache_systemr%r&r|rZ!ColorfulPyPrint_set_verbose_levelrZ%developer_enable_experimental_featurerirrhrZmy_host_name_urlencodedrrWrBrrUrZ_domainZhostnamerjrrrkZmy_host_scheme_escapedrZmyurl_prefix_escapedrr9r:reupdateZcustom_allowed_remote_headersZextract_root_domainrZmy_host_name_rootrZsteamed_mime_keywordsr~rrrrrrZ$enable_stream_transfer_async_preloadreZ$domains_whitelist_auto_add_glob_listrrZisolated_domainrrZ is_use_proxyrrZbuffrPZnetworkZ ip_networkrZquestionrrrrZ schedulerrr}rrrArCrSZREGEX_MY_HOST_NAMErZrrrrorrr\rorrsrtkrrrI__name__apprrrxrrrrrrfrZ$extract_from_url_may_have_extdomainsrZconvert_to_mirror_urlrrrrrr rr r&r*r>rrrErGrMrrVr_rlrhrdr{rrrirrrrrrrrrrrrrrrrZ after_requestrZrouterrrrrrrrrIrrfrgrrrrZcron_tasks_listZ _task_dictglobals ExceptionerYZthrZrr,r,r,r-s                                        '    9 $ / , @ 1  ) M    '       4 M)E * =6  4"= \'5@.   x K