§
    ·¯¸j†U ã                   ó¢L  — d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
mZmZmZmZmZmZmZmZ 	 ddl	mZ n# e$ r	 ddlmZ Y nw xY wddlZddlmZ ddlmc mZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$ dd	l%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? d
dl@mAZA d
dlBmCZC d
dlDmEZE d
dlFmGZGmHZHmIZImJZJ d
dlKmLZLmMZMmNZN dgZO ejP        eQ¦  «        ZRe'e'e(dœZS	 	 	 	 	 	 	 	 �dÝde>deTdeTdeUdeUdeUdeUdeVdeVdeeejW                          deTd ejW        fd!„ZX G d"„ d#ejW        ¦  «        ZY G d$„ d%ejW        ¦  «        ZZ G d&„ d'ejW        ¦  «        Z[ G d(„ d)ejW        ¦  «        Z\ G d*„ d+ejW        ¦  «        Z]	 	 	 �dÞd-ej^        d.e_d/eTd0eUfd1„Z` G d2„ dejW        ¦  «        Za�dßd4ejW        d5e_d6eUd dfd7„Zb	 	 	 �dàd4ejW        d5e_d8eVd6eUd df
d9„Zc�dßd4ejW        d5e_d6eUd dfd:„Zd�dßd4ejW        d5e_d6eUd dfd;„Ze�dád=e_d8eVd6eUd efd>„Zf	 	 	 	 �dâdAej^        dBej^        d/eTdCeeTeTf         dDe_dEeUd ej^        fdF„Zg ejh        ¦   «         �dãdGejW        dHe_dIe_dJeUd df
dK„¦   «         Zi	 �dädMee_ej^        f         dGeadIe_d ee_ej^        f         fdN„ZjdMee_ej^        f         dGead ee_ej^        f         fdO„ZkdMee_ej^        f         dGead ee_ej^        f         fdP„ZldMemfdQ„Zn	 	 	 �dådMee_ej^        f         dGeadReUdDe_dEeUd ee_ej^        f         fdS„Zo�dædTe_d ee_e
f         fdU„Zpi dV epdW¬X¦  «        “dY epdW¬X¦  «        “dZ epd[dWd¬\¦  «        “d] epd^dWdd_d`¬a¦  «        “db epdcdWd¬\¦  «        “dd epdedWdd_d`¬a¦  «        “df epdgdWd¬\¦  «        “dh epdidWdd_d`¬a¦  «        “dj epdkdWd¬\¦  «        “dl epdmdWdd_d`¬a¦  «        “dn epdodWd¬\¦  «        “dp epdqdWdd_d`¬a¦  «        “dr epdsdWd¬\¦  «        “dt epdudWd¬\¦  «        “dv epdwdWdd_d`¬a¦  «        “dx epdydW¬z¦  «        “d{ epd|dWd_d`¬}¦  «        “i d~ epddWd_d`¬}¦  «        “d€ epd�dWd¬\¦  «        “d‚ epdƒdWdd_d`¬a¦  «        “d„ epd…dWd¬\¦  «        “d† epd‡dWdd_d`¬a¦  «        “dˆ epd‰dWd¬\¦  «        “dŠ epd‹dWdd_d`¬a¦  «        “dŒ epd3¬�¦  «        “dŽ epd3¬�¦  «        “d� epd3¬�¦  «        “d� epd3¬�¦  «        “d‘ epdWd¬’¦  «        “d“ epdWd¬’¦  «        “d” epdWd¬’¦  «        “d• epdWd¬’¦  «        “d– epdWd¬’¦  «        “d— epd˜dWdd™¬š¦  «        “¥i d› epdœdWdd™¬š¦  «        “d� epdždWdd™¬š¦  «        “dŸ epd dWdd™¬š¦  «        “d¡ epd¢dWdd™¬š¦  «        “d£ epd¤dWdd™¬š¦  «        “d¥ epd¦dWdd™¬š¦  «        “d§ epd¨ddW¬©¦  «        “dª epd«ddW¬©¦  «        “d¬ epd­dWee d¬®¦  «        “d¯ epd°dWee d¬®¦  «        “d± epd²dWee d¬®¦  «        “d³ epd´dWee d¬®¦  «        “dµ epd¶dWd·ee dd¸d`¬¹¦  «        “dº epd»dWd·ee dd¸d`¬¹¦  «        “d¼ epd½dWd·ee dd¸d`¬¹¦  «        “d¾ epd¿dWd·ee dd¸d`¬¹¦  «        “dÀ epdÁdWd·ee dd¸d`¬¹¦  «        “¥i dÂ epdÃdWd·ee dd¸d`¬¹¦  «        “dÄ epdÅdWd·ee dd¸d`¬¹¦  «        “dÆ epdÇdWd·ee dd¸d`¬¹¦  «        “dÈ epdWd·dÉdÊddËd`¬Ì¦  «        “dÍ epdWd·dÉdÊddËd`¬Ì¦  «        “dÎ epdWd·dÉdÊddËd`¬Ì¦  «        “dÏ epdWd·dÉdÊddËd`¬Ì¦  «        “dÐ epdÑdWdÉdÊdÒdÓdÔ¬Õ¦  «        “dÖ epd×dWdÉdÊdÒdÓ¬Ø¦  «        “dÙ epdÚdW¬z¦  «        “dÛ epdWdÜdÝdÞ¬ß¦  «        “dà epdWdádÝ¬â¦  «        “dã epdWd_dÝdä¬å¦  «        “dæ epdádÝ¬ç¦  «        “dè ep¦   «         “dé epdWe#e$¬ê¦  «        “dë epdWe#e$d`d_¬ì¦  «        “¥i dí epdWe#e$d`dË¬ì¦  «        “dî epdWe#e$dÝ¬ï¦  «        “dð epdWe#e$d`d_dä¬ñ¦  «        “dò epdWe!e"d`¬ï¦  «        “dó epdWe!e"d`dôdä¬ñ¦  «        “dõ epdWe#e$d`¬ï¦  «        “dö epdWe#e$d`dôdä¬ñ¦  «        “d÷ epe#e$¬ø¦  «        “dù epdWe#e$dÝd_dä¬ñ¦  «        “dú epdWe#e$dÝ¬ï¦  «        “dû epdWe#e$dÝd_dä¬ñ¦  «        “dü epdWe#e$d`¬ï¦  «        “dý epdWe#e$d`dôdä¬ñ¦  «        “dþ epdWe#e$¬ê¦  «        “dÿ epdWe#e$d`¬ï¦  «        “�d  epdWe#e$d`d_dä¬ñ¦  «        “�d epdWe!e"d`¬ï¦  «        “¥i �d epdWe!e"d`dôdä¬ñ¦  «        “�d epdWe#e$d`¬ï¦  «        “�d epd3e#e$d`dôdä¬ñ¦  «        “�d epdWe#e$¬ê¦  «        “�d epdWe#e$¬ê¦  «        “�d epdWe#e$d`d_dä¬ñ¦  «        “�d epdWe#e$d`¬ï¦  «        “�d	 epdWe#e$dÞ�¬
¦  «        “�d epdWe!e"d`dÞ�¬¦  «        “�d epdWe#e$d`dÞ�¬¦  «        “�d epdWe#e$dÞ�¬
¦  «        “�d epdWe#e$d`dÞ�¬¦  «        “�d epdWe#e$�d�¬
¦  «        “�d epdWe#e$d`�d�¬¦  «        “�d epdWe!e"d`�d�¬¦  «        “�d epdWe#e$d`�d�¬¦  «        “�d epdWe#e$d`�d�¬¦  «        “¥i �d epdWe#e$d`�d�¬¦  «        “�d epdW�d�de#e$�d�¬¦  «        “�d epdW�de#e$d`�d�¬¦  «        “�d  epdW�de#e$dÜd`�d!�¬"¦  «        “�d# epdW�de#e$d`�d�¬¦  «        “�d$ epdWe#e$d`�d�¬¦  «        “�d% epdWe#e$d`dá�d�¬&¦  «        “�d' epdWe#e$d`�d�¬¦  «        “�d( epdWe#e$d`�d�¬¦  «        “�d) epdW�d*e#e$d`�d�¬¦  «        “�d+ epdW�d*e#e$d`�d�¬¦  «        “�d, epdW�d*�de#e$d`�d�¬-¦  «        “�d. epdW�d*�de#e$d`�d�¬-¦  «        “�d/ epdWe#e$�d*�dd`�d0�d�¬1¦  «        “�d2 epdW�d3�de#e$d`�d�¬-¦  «        “�d4 epdW�d3e#e$�d0d`dä�d�¬5¦  «        “�d6 epdW�d3e#e$d`�d�¬¦  «        “¥i �d7 epdW�d3e#e$�d0d`dä�d�¬5¦  «        “�d8 epdW�d3�de#e$d`�d�¬-¦  «        “�d9 epdW�d3�de#e$d`�d�¬-¦  «        “�d: epdW�d3�de#e$d`�d�¬-¦  «        “�d; epdW�d3�de#e$d`�d�¬-¦  «        “�d< epdW�d3e#e$d`�d�¬¦  «        “�d= epdW�d3�de#e$d`�d�¬-¦  «        “�d> epdW�d3�de#e$d`�d�¬-¦  «        “�d? epdW�d3�de#e$d`�d�¬-¦  «        “�d@ epdW�d3�de#e$d`�d�¬-¦  «        “�dA epdW�de#e$�d�¬B¦  «        “�dC epdW�de#e$�d�¬B¦  «        “�dD epdW�de#e$d`�d�¬E¦  «        “�dF epdW�de#e$d`dô�d�¬G¦  «        “�dH epdW�de#e$d`�dI�¬J¦  «        “�dK epd3dádÝ�¬L¦  «        “�dM epd3dÜdÝ�¬L¦  «        “¥i �dN epd3¬�¦  «        “�dO epd3¬�¦  «        “�dP epd3¬�¦  «        “�dQ epdW�de#e$�dRd`�¬S¦  «        “�dT epdW�de#e$dôd`dä�¬U¦  «        “�dV epdW�de#e$�dRd`�¬S¦  «        “�dW epdW�de#e$dôd`dä�¬U¦  «        “�dX ep�dYddWdÜdÝ�¬Z¦  «        “�d[ ep�d\ddWdÜdÝ�¬Z¦  «        “�d] ep�d^ddWdÜdÝ�¬Z¦  «        “�d_ ep�d`ddWdÜdÝ�¬Z¦  «        “�da ep�dbddWdÜdÝ�¬Z¦  «        “�dc ep�ddddWdÜdÝ�¬Z¦  «        “�de ep�dfddWdÜdÝd™�¬g¦  «        “�dh ep�diddWdÜdÝd™�¬g¦  «        “�dj ep�dkddWdÜdÝ�¬Z¦  «        “�dl ep�dmddWdÜdÝ�¬Z¦  «        “¥i �dn ep�doddWdÜdÝ�¬Z¦  «        “�dp ep�dqddWdÜdÝd™�¬g¦  «        “�dr ep�dsddWdÜdÝd™�¬g¦  «        “�dt epd3¬�¦  «        “�du epd3¬�¦  «        “�dv epd3¬�¦  «        “�dw ep�dxdW�d3ee d�¬y¦  «        “�dz ep�d{dW�d3ee d�¬y¦  «        “�d| ep�d}dW�d3ee d�¬y¦  «        “�d~ ep�d�d3ee d�¬€¦  «        “�d� ep�d‚�d3ee d�¬€¦  «        “�dƒ ep�d„�d3dËd`ee d�¬…¦  «        “�d† ep�d‡�d3ee d�¬€¦  «        “�dˆ epdWdád�¬‰¦  «        “�dŠ epdWd¬’¦  «        “�d‹ epdWd¬’¦  «        “�dŒ epdWdád�¬‰¦  «        “¥i �d� epdWdád�¬‰¦  «        “�dŽ epdWdád�¬‰¦  «        “�d� epdWd_d�¬‰¦  «        “�d� epdWd_d�¬‰¦  «        “�d‘ epdW�d’d�¬‰¦  «        “�d“ epdW�d’d�¬‰¦  «        “�d” epdWdád�¬‰¦  «        “�d• epdWdád�¬‰¦  «        “�d– epdWd_d�¬‰¦  «        “�d— epdWd_d�¬‰¦  «        “�d˜ epdW�d’d�¬‰¦  «        “�d™ epdWd¬’¦  «        “�dš epdWd¬’¦  «        “�d› epdW�d0d�¬‰¦  «        “�dœ epdW�d0d�¬‰¦  «        “�d� epdWd_d�¬‰¦  «        “�dž epdWdád�¬‰¦  «        “¥i �dŸ epdWdád�¬‰¦  «        “�d  epdWd_d�¬‰¦  «        “�d¡ epdW�d’d�¬‰¦  «        “�d¢ epdWdád�¬‰¦  «        “�d£ epdWd_d�¬‰¦  «        “�d¤ epdWdád�¬‰¦  «        “�d¥ epdWd¬’¦  «        “�d¦ epdWd¬’¦  «        “�d§ epdWdád�¬‰¦  «        “�d¨ epdWdád�¬‰¦  «        “�d© epdWdád�¬‰¦  «        “�dª epdWd_d�¬‰¦  «        “�d« epdWd_d�¬‰¦  «        “�d¬ epdW�d’d�¬‰¦  «        “�d­ epdW�d’d�¬‰¦  «        “�d® epdWdád�¬‰¦  «        “�d¯ epdWdád�¬‰¦  «        “¥i �d° epdWd_d�¬‰¦  «        “�d± epdWd_d�¬‰¦  «        “�d² epdW�d’d�¬‰¦  «        “�d³ epdWd¬’¦  «        “�d´ epdWd¬’¦  «        “�dµ epdWd¬’¦  «        “�d¶ epdWd¬’¦  «        “�d· epdWd¬’¦  «        “�d¸ epdWd¬’¦  «        “�d¹ epdW�d0d�¬‰¦  «        “�dº epdW�d0d`d¬ß¦  «        “�d» epdWd_d`d¬ß¦  «        “�d¼ epdWdËd`d¬ß¦  «        “�d½ epdWdËd`d¬ß¦  «        “�d¾ epdWdËd`d¬ß¦  «        “�d¿ epdWdËd`d¬ß¦  «        “�dÀ epdWdËd`d¬ß¦  «        “¥i �dÁ epdWdËd`d¬ß¦  «        “�dÂ epdWdËd`d¬ß¦  «        “�dÃ epdWdËd`d¬ß¦  «        “�dÄ epdW�dÅd`d¬ß¦  «        “�dÆ epdW�dÅd`d¬ß¦  «        “�dÇ epdW�dÅd`d¬ß¦  «        “�dÈ epdW�dÅd`d¬ß¦  «        “�dÉ epdW�dÅd`d¬ß¦  «        “�dÊ epdWdád�¬‰¦  «        “�dË epdWdád�¬‰¦  «        “�dÌ epdWd_d�¬‰¦  «        “�dÍ epdW�d’d�¬‰¦  «        “�dÎ epdWdád�¬‰¦  «        “�dÏ epdWd_d�¬‰¦  «        “�dÐ epdW�d0d`dä¬å¦  «        “�dÑ epdW�d0d`dä¬å¦  «        “�dÒ epdW�de#e$�d�¬Ó¦  «        “¥i �dÔ epdW�de#e$�d�¬Ó¦  «        “�dÕ epdW�de#e$�d�¬Ó¦  «        “�dÖ epdW�de#e$�d�¬Ó¦  «        “�d× epdWdádÝ¬â¦  «        “�dØ epdWdádÝ¬â¦  «        “�dÙ epdWdádÝ¬â¦  «        “�dÚ epdWdádÝ¬â¦  «        “�dÛ epdWdádÝ¬â¦  «        “�dÜ epdWdádÝ¬â¦  «        “�dÝ epdWdádÝ¬â¦  «        “�dÞ epdWdÞdádÝ�¬ß¦  «        “�dà epdWdádÝ¬â¦  «        “�dá epdWdádÝ¬â¦  «        “�dâ epdWdádÝ¬â¦  «        “�dã epdWdádÝ¬â¦  «        “�dä epdWdádÝ¬â¦  «        “�då epdWdÞdádÝ�¬ß¦  «        “¥i �dæ epdWdádÝ¬â¦  «        “�dç epdWdádÝ¬â¦  «        “�dè epdWdÞdádÝ�¬ß¦  «        “�dé epdWdÞdádÝ�¬ß¦  «        “�dê epdWd_d`¬â¦  «        “�dë epdWdádÝ¬â¦  «        “�dì epdWdádÝ¬â¦  «        “�dí epdWdádÝ¬â¦  «        “�dî epdWdádÝ¬â¦  «        “�dï epdWdÞdádÝ�¬ß¦  «        “�dð epdWdÞdádÝ�¬ß¦  «        “�dñ epdWd_d`¬â¦  «        “�dò epdá�¬ó¦  «        “�dô epdWdádÝ¬â¦  «        “�dõ epdWdÞdádÝ�¬ß¦  «        “�dö epdWd_d`¬â¦  «        “�d÷ epdá�¬ó¦  «        “¥i �dø epdWdád`¬â¦  «        “�dù epdWdÞdád`�¬ß¦  «        “�dú epdWd_d`¬â¦  «        “�dû epdWdËd`dä¬å¦  «        “�dü epdW�dee dËd`d�¬"¦  «        “�dý epdWee dËd`d�¬þ¦  «        “�dÿ epdWe#e$�d*d`d�¬ ¦  «        “�d epdWe#e$�d*d`d�¬ ¦  «        “�d epdWe#e$�d*d`d�¬ ¦  «        “�d epdWe#e$�d*d`d�¬ ¦  «        “�d epdWe#e$�d*d`d�¬ ¦  «        “�d epdWe#e$�d*dôd`d�¬¦  «        “�d epdWe#e$�d*dôd`d�¬¦  «        “�d epdWe#e$�d*dôd`d�¬¦  «        “�d	 epdWe#e$�d*dôd`d�¬¦  «        “�d
 epdWe#e$�d*dôd`d�¬¦  «        “�d epdWe#e$�d*dËd`d�¬¦  «        “¥i �d epdWe#e$�d*dËd`d�¬¦  «        “�d epdWe#e$�d*dËd`d�¬¦  «        “�d epdWe#e$�d*dËd`d�¬¦  «        “�d epdW�ddÝ¬â¦  «        “�d epdW�ddÝ¬â¦  «        “�d epdW�ddÝ¬â¦  «        “�d ep�ddÝ¬ç¦  «        “�d epdWee d`¬ï¦  «        “�d epdWee d`¬ï¦  «        “�d epdWee d`¬ï¦  «        “�d epdWee d`¬ï¦  «        “�d epd3ee d`�¬¦  «        “�d epd3dôee d`�¬¦  «        “�d epdWee d`d�¬¦  «        “�d epdWee d`d�¬¦  «        “�d epdWee d`d�¬¦  «        “�d epdWee d`d�¬¦  «        “¥Zq�d „ eq r                    ¦   «         D ¦   «         ZsesD ]IZt eju        eqet         ¦  «        Zvev�d!         dWk    r	dWetz   ev�d!<   eveqet w                    �d"�d#¦  «        <   ŒJ eLeq¦  «        Zqejx         y                    �d$�d%¦  «         z                    ¦   «         �d&k    Z{	 	 �dç�d'e_�d(eU�d)eeU         d eea�d*f         f�d+„Z|eM�dè�d(eUd eaf�d,„¦   «         Z}eM�dè�d(eUd eaf�d-„¦   «         Z~eM�dè�d(eUd eaf�d.„¦   «         ZeM�dè�d(eUd eaf�d/„¦   «         Z€eM�dè�d(eUd eaf�d0„¦   «         Z�eM�dè�d(eUd eaf�d1„¦   «         Z‚eM�dè�d(eUd eaf�d2„¦   «         ZƒeM�dè�d(eUd eaf�d3„¦   «         Z„eM�dè�d(eUd eaf�d4„¦   «         Z…eM�dè�d(eUd eaf�d5„¦   «         Z†eM�dè�d(eUd eaf�d6„¦   «         Z‡eM�dè�d(eUd eaf�d7„¦   «         ZˆeM�dè�d(eUd eaf�d8„¦   «         Z‰eM�dè�d(eUd eaf�d9„¦   «         ZŠeM�dè�d(eUd eaf�d:„¦   «         Z‹eM�dè�d(eUd eaf�d;„¦   «         ZŒeM�dè�d(eUd eaf�d<„¦   «         Z�eM�dè�d(eUd eaf�d=„¦   «         ZŽeM�dè�d(eUd eaf�d>„¦   «         Z�eM�dè�d(eUd eaf�d?„¦   «         Z�eM�dè�d(eUd eaf�d@„¦   «         Z‘eM�dè�d(eUd eaf�dA„¦   «         Z’eM�dè�d(eUd eaf�dB„¦   «         Z“eM�dè�d(eUd eaf�dC„¦   «         Z”eM�dè�d(eUd eaf�dD„¦   «         Z•eM�dè�d(eUd eaf�dE„¦   «         Z–eM�dè�d(eUd eaf�dF„¦   «         Z—eM�dè�d(eUd eaf�dG„¦   «         Z˜eM�dè�d(eUd eaf�dH„¦   «         Z™eM�dè�d(eUd eaf�dI„¦   «         ZšeM�dè�d(eUd eaf�dJ„¦   «         Z›eM�dè�d(eUd eaf�dK„¦   «         ZœeM�dè�d(eUd eaf�dL„¦   «         Z�eM�dè�d(eUd eaf�dM„¦   «         ZžeM�dè�d(eUd eaf�dN„¦   «         ZŸeM�dè�d(eUd eaf�dO„¦   «         Z eM�dè�d(eUd eaf�dP„¦   «         Z¡eM�dè�d(eUd eaf�dQ„¦   «         Z¢eM�dè�d(eUd eaf�dR„¦   «         Z£eM�dè�d(eUd eaf�dS„¦   «         Z¤eM�dè�d(eUd eaf�dT„¦   «         Z¥eM�dè�d(eUd eaf�dU„¦   «         Z¦eM�dè�d(eUd eaf�dV„¦   «         Z§eM�dè�d(eUd eaf�dW„¦   «         Z¨eM�dè�d(eUd eaf�dX„¦   «         Z©eM�dè�d(eUd eaf�dY„¦   «         ZªeM�dè�d(eUd eaf�dZ„¦   «         Z«eM�dè�d(eUd eaf�d[„¦   «         Z¬eM�dè�d(eUd eaf�d\„¦   «         Z­eM�dè�d(eUd eaf�d]„¦   «         Z®eM�dè�d(eUd eaf�d^„¦   «         Z¯eM�dè�d(eUd eaf�d_„¦   «         Z°eM�dè�d(eUd eaf�d`„¦   «         Z±eM�dè�d(eUd eaf�da„¦   «         Z²eM�dè�d(eUd eaf�db„¦   «         Z³eM�dè�d(eUd eaf�dc„¦   «         Z´eM�dè�d(eUd eaf�dd„¦   «         ZµeM�dè�d(eUd eaf�de„¦   «         Z¶eM�dè�d(eUd eaf�df„¦   «         Z·eM�dè�d(eUd eaf�dg„¦   «         Z¸eM�dè�d(eUd eaf�dh„¦   «         Z¹eM�dè�d(eUd eaf�di„¦   «         ZºeM�dè�d(eUd eaf�dj„¦   «         Z»eM�dè�d(eUd eaf�dk„¦   «         Z¼eM�dè�d(eUd eaf�dl„¦   «         Z½eM�dè�d(eUd eaf�dm„¦   «         Z¾eM�dè�d(eUd eaf�dn„¦   «         Z¿eM�dè�d(eUd eaf�do„¦   «         ZÀeM�dè�d(eUd eaf�dp„¦   «         ZÁeM�dè�d(eUd eaf�dq„¦   «         ZÂeM�dè�d(eUd eaf�dr„¦   «         ZÃeM�dè�d(eUd eaf�ds„¦   «         ZÄeM�dè�d(eUd eaf�dt„¦   «         ZÅeM�dè�d(eUd eaf�du„¦   «         ZÆeM�dè�d(eUd eaf�dv„¦   «         ZÇeM�dè�d(eUd eaf�dw„¦   «         ZÈeM�dè�d(eUd eaf�dx„¦   «         ZÉeM�dè�d(eUd eaf�dy„¦   «         ZÊeM�dè�d(eUd eaf�dz„¦   «         ZËeM�dè�d(eUd eaf�d{„¦   «         ZÌeM�dè�d(eUd eaf�d|„¦   «         ZÍeM�dè�d(eUd eaf�d}„¦   «         ZÎeM�dè�d(eUd eaf�d~„¦   «         ZÏeM�dè�d(eUd eaf�d„¦   «         ZÐeM�dè�d(eUd eaf�d€„¦   «         ZÑeM�dè�d(eUd eaf�d�„¦   «         ZÒeM�dè�d(eUd eaf�d‚„¦   «         ZÓeM�dè�d(eUd eaf�dƒ„¦   «         ZÔeM�dè�d(eUd eaf�d„„¦   «         ZÕeM�dè�d(eUd eaf�d…„¦   «         ZÖeM�dè�d(eUd eaf�d†„¦   «         Z×eM�dè�d(eUd eaf�d‡„¦   «         ZØeM�dè�d(eUd eaf�dˆ„¦   «         ZÙeM�dè�d(eUd eaf�d‰„¦   «         ZÚeM�dè�d(eUd eaf�dŠ„¦   «         ZÛeM�dè�d(eUd eaf�d‹„¦   «         ZÜeM�dè�d(eUd eaf�dŒ„¦   «         ZÝeM�dè�d(eUd eaf�d�„¦   «         ZÞeM�dè�d(eUd eaf�dŽ„¦   «         ZßeM�dè�d(eUd eaf�d�„¦   «         ZàeM�dè�d(eUd eaf�d�„¦   «         ZáeM�dè�d(eUd eaf�d‘„¦   «         ZâeM�dè�d(eUd eaf�d’„¦   «         ZãeM�dè�d(eUd eaf�d“„¦   «         ZäeM�dè�d(eUd eaf�d”„¦   «         ZåeM�dè�d(eUd eaf�d•„¦   «         ZæeM�dè�d(eUd eaf�d–„¦   «         ZçeM�dè�d(eUd eaf�d—„¦   «         ZèeM�dè�d(eUd eaf�d˜„¦   «         ZéeM�dè�d(eUd eaf�d™„¦   «         ZêeM�dè�d(eUd eaf�dš„¦   «         ZëeM�dè�d(eUd eaf�d›„¦   «         ZìeM�dè�d(eUd eaf�dœ„¦   «         ZíeM�dè�d(eUd eaf�d�„¦   «         ZîeM�dè�d(eUd eaf�dž„¦   «         ZïeM�dè�d(eUd eaf�dŸ„¦   «         ZðeM�dè�d(eUd eaf�d „¦   «         ZñeM�dè�d(eUd eaf�d¡„¦   «         ZòeM�dè�d(eUd eaf�d¢„¦   «         ZóeM�dè�d(eUd eaf�d£„¦   «         ZôeM�dè�d(eUd eaf�d¤„¦   «         ZõeM�dè�d(eUd eaf�d¥„¦   «         ZöeM�dè�d(eUd eaf�d¦„¦   «         Z÷eM�dè�d(eUd eaf�d§„¦   «         ZøeM�dè�d(eUd eaf�d¨„¦   «         ZùeM�dè�d(eUd eaf�d©„¦   «         ZúeM�dè�d(eUd eaf�dª„¦   «         ZûeM�dè�d(eUd eaf�d«„¦   «         ZüeM�dè�d(eUd eaf�d¬„¦   «         ZýeM�dè�d(eUd eaf�d­„¦   «         ZþeM�dè�d(eUd eaf�d®„¦   «         ZÿeM�dè�d(eUd eaf�d¯„¦   «         �Z eM�dè�d(eUd eaf�d°„¦   «         �ZeM�dè�d(eUd eaf�d±„¦   «         �ZeM�dè�d(eUd eaf�d²„¦   «         �ZeM�dè�d(eUd eaf�d³„¦   «         �ZeM�dè�d(eUd eaf�d´„¦   «         �ZeM�dè�d(eUd eaf�dµ„¦   «         �ZeM�dè�d(eUd eaf�d¶„¦   «         �ZeM�dè�d(eUd eaf�d·„¦   «         �ZeM�dè�d(eUd eaf�d¸„¦   «         �Z	eM�dè�d(eUd eaf�d¹„¦   «         �Z
eM�dè�d(eUd eaf�dº„¦   «         �ZeM�dè�d(eUd eaf�d»„¦   «         �ZeM�dè�d(eUd eaf�d¼„¦   «         �ZeM�dè�d(eUd eaf�d½„¦   «         �ZeM�dè�d(eUd eaf�d¾„¦   «         �ZeM�dè�d(eUd eaf�d¿„¦   «         �ZeM�dè�d(eUd eaf�dÀ„¦   «         �ZeM�dè�d(eUd eaf�dÁ„¦   «         �ZeM�dè�d(eUd eaf�dÂ„¦   «         �ZeM�dè�d(eUd eaf�dÃ„¦   «         �ZeM�dè�d(eUd eaf�dÄ„¦   «         �ZeM�dè�d(eUd eaf�dÅ„¦   «         �ZeM�dè�d(eUd eaf�dÆ„¦   «         �ZeM�dè�d(eUd eaf�dÇ„¦   «         �ZeM�dè�d(eUd eaf�dÈ„¦   «         �Z eNeQi �dÉd—“�dÊd›“�dËd�“�dÌdŸ“�dÍd¡“�dÎd£“�dÏd”“�dÐd¥“�dÑd–“�dÒ�dÓ“�dÔ�dÕ“�dÖd¬“�d×d¯“�dØd±“�dÙd³“�dÚdÐ“�dÛ�d“�d�d�d�dÜœ¥¦  «         dS (é  aß   Vision Transformer (ViT) in PyTorch

A PyTorch implement of Vision Transformers as described in:

'An Image Is Worth 16 x 16 Words: Transformers for Image Recognition at Scale'
    - https://arxiv.org/abs/2010.11929

`How to train your ViT? Data, Augmentation, and Regularization in Vision Transformers`
    - https://arxiv.org/abs/2106.10270

`FlexiViT: One Model for All Patch Sizes`
    - https://arxiv.org/abs/2212.08013

The official jax code is released and available at
  * https://github.com/google-research/vision_transformer
  * https://github.com/google-research/big_vision

Acknowledgments:
  * The paper authors for releasing code and weights, thanks!
  * I fixed my class token impl based on Phil Wang's https://github.com/lucidrains/vit-pytorch
  * Simple transformer style inspired by Andrej Karpathy's https://github.com/karpathy/minGPT
  * Bert reference code checks against Huggingface Transformers and Tensorflow Bert

Hacked together by / Copyright 2020, Ross Wightman
é    N)ÚOrderedDict)Úpartial)	ÚAnyÚCallableÚDictÚOptionalÚSetÚTupleÚTypeÚUnionÚList)ÚLiteral)ÚFinal)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STDÚIMAGENET_INCEPTION_MEANÚIMAGENET_INCEPTION_STDÚOPENAI_CLIP_MEANÚOPENAI_CLIP_STD)Úget_device_dtypeÚ	AttentionÚDiffAttentionÚAttentionPoolLatentÚAttentionPoolPrrÚ
PatchEmbedÚMlpÚSwiGLUPackedÚSwiGLUÚ	LayerNormÚRmsNormÚDropPathÚcalculate_drop_path_ratesÚPatchDropoutÚtrunc_normal_Úlecun_normal_Úresample_patch_embedÚresample_abs_pos_embedÚresample_abs_pos_embed_nhwcÚuse_fused_attnÚget_act_layerÚget_norm_layerÚmaybe_add_maskÚresolve_self_attn_maskÚ	LayerTypeÚ
LayerScaleé   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú_load_npz_checkpoint)Únamed_applyÚ
checkpointÚcheckpoint_seqÚadapt_input_conv)Úgenerate_default_cfgsÚregister_modelÚregister_model_deprecationsÚVisionTransformer)Ú ÚattnÚdiffFTç        Ú
attn_layerÚdimÚ	num_headsÚqkv_biasÚqk_normÚ
scale_normÚ	proj_biasÚ	attn_dropÚ	proj_dropÚ
norm_layerÚdepthÚreturnc                 óØ   — t          | t          ¦  «        r*t                               | d ¦  «        } | €J d| › �¦   «         ‚t	          | t
          ¦  «        r|
|d<    | |f||||||||	dœ|¤ŽS )NzUnknown attn_layer: rJ   )rB   rC   rD   rE   rF   rG   rH   rI   )Ú
isinstanceÚstrÚATTN_LAYERSÚgetÚ
issubclassr   )r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   Úkwargss               ún/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/models/vision_transformer.pyÚ_create_attnrT   _   s§   € õ �*�cÑ"Ô"ð KÝ —_’_ Z°Ñ6Ô6ˆ
ØÐ%Ð%Ð'J¸jÐ'JÐ'JÑ%Ô%Ð%õ �*�mÑ,Ô,ð  Øˆˆw‰àˆ:ØðàØØØØØØØðð ð ðð ð ó    c            %       ó0  ‡ — e Zd ZdZddddddddddej        eeedddfde	d	e	d
e
dededededede
de
dee
         de
deej                 deej                 deej                 dede	ddf$ˆ fd„Z	 	 ddej        deej                 dedej        fd„Zˆ xZS ) ÚBlockz)Transformer block with pre-normalization.ç      @FTr?   Nr   rA   rB   Ú	mlp_ratiorC   rD   Úscale_attn_normÚscale_mlp_normrF   rH   rG   Úinit_valuesÚ	drop_pathÚ	act_layerrI   Ú	mlp_layerr@   rJ   rK   c                 ól  •— t          ¦   «                              ¦   «          ||dœ} ||fi |¤Ž| _        t          ||f||||||
|	||dœ	|¤Ž| _        |rt          |fd|i|¤Žnt          j        ¦   «         | _        |dk    rt          |¦  «        nt          j        ¦   «         | _
         ||fi |¤Ž| _         |d|t          ||z  ¦  «        ||r|nd||	dœ|¤Ž| _        |rt          |fd|i|¤Žnt          j        ¦   «         | _        |dk    rt          |¦  «        nt          j        ¦   «         | _        dS )af  Initialize Block.

        Args:
            dim: Number of input channels.
            num_heads: Number of attention heads.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: If True, add a learnable bias to query, key, value.
            qk_norm: If True, apply normalization to query and key.
            proj_bias: If True, add bias to output projection.
            proj_drop: Projection dropout rate.
            attn_drop: Attention dropout rate.
            init_values: Initial values for layer scale.
            drop_path: Stochastic depth rate.
            act_layer: Activation layer.
            norm_layer: Normalization layer.
            mlp_layer: MLP layer.
            attn_layer: Attention layer type (class or string).
            depth: Block index, passed to attention layer for depth-dependent init.
        ©ÚdeviceÚdtype©	rB   rC   rD   rE   rF   rG   rH   rI   rJ   r\   r?   N©Úin_featuresÚhidden_featuresr^   rI   ÚbiasÚdrop© )ÚsuperÚ__init__Únorm1rT   r=   r/   ÚnnÚIdentityÚls1r!   Ú
drop_path1Únorm2ÚintÚmlpÚls2Ú
drop_path2©ÚselfrA   rB   rY   rC   rD   rZ   r[   rF   rH   rG   r\   r]   r^   rI   r_   r@   rJ   rb   rc   ÚddÚ	__class__s                        €rS   rl   zBlock.__init__†   s¡  ø€ õR 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ý ØØð
ð  ØØØ&ØØØØ!Øð
ð 
ð ð
ð 
ˆŒ	ð FQÐc•:˜cÐAÐA¨{ÐA¸bÐAÐAÐAÕVXÔVaÑVcÔVcˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�Z Ð*Ð* rÐ*Ð*ˆŒ
Ø�9ð 
ØÝ  i¡Ñ0Ô0ØØ%3Ð=�z�z¸ØØð
ð 
ð ð
ð 
ˆŒð FQÐc•:˜cÐAÐA¨{ÐA¸bÐAÐAÐAÕVXÔVaÑVcÔVcˆŒØ1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒˆˆrU   ÚxÚ	attn_maskÚ	is_causalc           
      óP  — ||                       |                      |                      |                      |¦  «        ||¬¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |                      |¦  «        ¦  «        ¦  «        ¦  «        z   }|S ©N©r|   r}   )rq   rp   r=   rm   rv   ru   rt   rr   ©rx   r{   r|   r}   s       rS   ÚforwardzBlock.forwardÑ   s€   € ð �—’ §¢¨¯ª°4·:²:¸a±=´=ÈIÐaj¨Ñ)kÔ)kÑ lÔ lÑmÔmÑmˆØ�—’ §¢¨¯ª°$·*²*¸Q±-´-Ñ)@Ô)@Ñ AÔ AÑBÔBÑBˆØˆrU   ©NF©Ú__name__Ú
__module__Ú__qualname__Ú__doc__rn   ÚGELUr   r   r   rs   ÚfloatÚboolr   r   ÚModuler.   rl   ÚtorchÚTensorr‚   Ú__classcell__©rz   s   @rS   rW   rW   ƒ   sÌ  ø€ € € € € Ø3Ð3ð  "Ø"Ø!Ø$)Ø#(Ø"Ø!Ø!Ø+/Ø!Ø)+¬Ø*3Ø),Ø$-ØØØð)ISð ISàðISð ðISð ð	ISð
 ðISð ðISð "ðISð !ðISð ðISð ðISð ðISð " %œðISð ðISð ˜BœI”ðISð ˜RœYœðISð  ˜BœI”ð!ISð" "ð#ISð$ ð%ISð* 
ð+ISð ISð ISð ISð ISð ISð\ 15Ø#ð	ð àŒ|ðð   ¤Ô-ðð ð	ð
 
Œðð ð ð ð ð ð ð rU   rW   c            %       ó4  ‡ — e Zd Zddddddddddej        eeedddfdeded	e	d
e
de
de
de
de
de	de	dee	         de	deej                 deej                 deej                 dededdf$ˆ fd„Zdd„Z	 	 d dej        deej                 de
dej        fd„Zˆ xZS )!ÚResPostBlockrX   FTr?   Nr   rA   rB   rY   rC   rD   rZ   r[   rF   rH   rG   r\   r]   r^   rI   r_   r@   rJ   rK   c                 óþ  •— t          ¦   «                              ¦   «          ||dœ}|| _        t          ||f||||||
|	||dœ	|¤Ž| _         ||fi |¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _	         |d|t          ||z  ¦  «        ||r|nd ||	dœ|¤Ž| _         ||fi |¤Ž| _        |dk    rt          |¦  «        nt          j        ¦   «         | _        |                      ¦   «          d S )Nra   rd   r?   re   rj   )rk   rl   r\   rT   r=   rm   r!   rn   ro   rq   rs   rt   rr   rv   Úinit_weightsrw   s                        €rS   rl   zResPostBlock.__init__Ý   sX  ø€ õ, 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ&ˆÔå ØØð
ð  ØØØ&ØØØØ!Øð
ð 
ð ð
ð 
ˆŒ	ð  �Z Ð*Ð* rÐ*Ð*ˆŒ
Ø1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà�9ð 
ØÝ  i¡Ñ0Ô0ØØ%3Ð=�z�z¸ØØð
ð 
ð ð
ð 
ˆŒð  �Z Ð*Ð* rÐ*Ð*ˆŒ
Ø1:¸R²°�( 9Ñ-Ô-Ð-ÅRÄ[Á]Ä]ˆŒà×ÒÑÔÐÐÐrU   c                 óÔ   — | j         �`t          j                             | j        j        | j         ¦  «         t          j                             | j        j        | j         ¦  «         d S d S ©N)r\   rn   ÚinitÚ	constant_rm   Úweightrr   ©rx   s    rS   r”   zResPostBlock.init_weights  sY   € àÔÐ'ÝŒG×Ò˜dœjÔ/°Ô1AÑBÔBÐBÝŒG×Ò˜dœjÔ/°Ô1AÑBÔBÐBÐBÐBð (Ð'rU   r{   r|   r}   c           
      ó  — ||                       |                      |                      |||¬¦  «        ¦  «        ¦  «        z   }||                      |                      |                      |¦  «        ¦  «        ¦  «        z   }|S r   )rq   rm   r=   rv   rr   rt   r�   s       rS   r‚   zResPostBlock.forward  sj   € ð �—’ §
¢
¨4¯9ª9°QÀ)ÐW`¨9Ñ+aÔ+aÑ bÔ bÑcÔcÑcˆØ�—’ §
¢
¨4¯8ª8°A©;¬;Ñ 7Ô 7Ñ8Ô8Ñ8ˆØˆrU   ©rK   Nrƒ   )r…   r†   r‡   rn   r‰   r   r   r   rs   rŠ   r‹   r   r   rŒ   r.   rl   r”   r�   rŽ   r‚   r�   r�   s   @rS   r’   r’   Ü   s¬  ø€ € € € € ð
  "Ø"Ø!Ø$)Ø#(Ø"Ø!Ø!Ø+/Ø!Ø)+¬Ø*3Ø),Ø$-ØØØð)7ð 7àð7ð ð7ð ð	7ð
 ð7ð ð7ð "ð7ð !ð7ð ð7ð ð7ð ð7ð " %œð7ð ð7ð ˜BœI”ð7ð ˜RœYœð7ð  ˜BœI”ð!7ð" "ð#7ð$ ð%7ð* 
ð+7ð 7ð 7ð 7ð 7ð 7ðrCð Cð Cð Cð 15Ø#ð	ð àŒ|ðð   ¤Ô-ðð ð	ð
 
Œðð ð ð ð ð ð ð rU   r’   c            '       ón  ‡ — e Zd ZU dZee         ed<   ddddddddddej        e	ddddddfd	e
d
e
dededededededededee         dedeej                 deej                 deeej                          dee         de
deddf&ˆ fd„Zd"d„Z	 	 d#dej        deej                 d edej        fd!„Zˆ xZS )$ÚParallelScalingBlockz¤ Parallel ViT block (MLP & Attention in parallel)
    Based on:
      'Scaling Vision Transformers to 22 Billion Parameters` - https://arxiv.org/abs/2302.05442
    Ú
fused_attnrX   FTr?   Nr   rA   rB   rY   rC   rD   rZ   r[   rF   rH   rG   r\   r]   r^   rI   r_   r@   rJ   Úfuse_out_projrK   c                 óè  •— t          ¦   «                              ¦   «          ||dœ}||z  dk    s
J d¦   «         ‚|s|r
J d¦   «         ‚|| _        ||z  | _        | j        dz  | _        t          ¦   «         | _        t          ||z  ¦  «        }|d|z  z   } ||fi |¤Ž| _        t          j
        ||fd|i|¤Ž| _        |g|gdz  z   | _        |r|                      dd ¦  «         n)t          j        t          j        |fi |¤Ž¦  «        | _        |r || j        fi |¤Žnt          j        ¦   «         | _        |r || j        fi |¤Žnt          j        ¦   «         | _        t          j        |
¦  «        | _        t          j        |	¦  «        | _         |¦   «         | _        |r,t          j
        ||z   |fd|i|¤Ž| _        d | _        d | _        n;d | _        t          j
        ||fd|i|¤Ž| _        t          j
        ||fd|i|¤Ž| _        |�t9          |fd	|i|¤Žnt          j        ¦   «         | _        |d
k    rt=          |¦  «        nt          j        ¦   «         | _        |                       ¦   «          d S )Nra   r   ú$dim should be divisible by num_headsúScale norms not supportedç      à¿é   rh   Úmlp_biasr\   r?   )!rk   rl   rB   Úhead_dimÚscaler)   rŸ   rs   Úin_normrn   ÚLinearÚin_projÚin_splitÚregister_parameterÚ	Parameterr�   Úemptyr¦   ro   Úq_normÚk_normÚDropoutrG   Úmlp_dropÚmlp_actÚout_projÚattn_out_projÚmlp_out_projr/   Úlsr!   r]   Úreset_parameters)rx   rA   rB   rY   rC   rD   rZ   r[   rF   rH   rG   r\   r]   r^   rI   r_   r@   rJ   r    rb   rc   ry   Úmlp_hidden_dimÚin_proj_out_dimrz   s                           €rS   rl   zParallelScalingBlock.__init__.  s¿  ø€ õ. 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ�Y‰ !Ò#Ð#Ð#Ð%KÑ#Ô#Ð#Ø"ÐV¨>ÐVÐVÐ;VÑVÔVÐ9Ø"ˆŒØ˜yÑ(ˆŒØ”] dÑ*ˆŒ
Ý(Ñ*Ô*ˆŒÝ˜Y¨™_Ñ-Ô-ˆØ(¨1¨s©7Ñ2ˆà!�z #Ð,Ð,¨Ð,Ð,ˆŒÝ”y  oÐKÐK¸HÐKÈÐKÐKˆŒØ'Ð(¨C¨5°1©9Ñ4ˆŒØð 	Là×#Ò# J°Ñ5Ô5Ð5Ð5åœL­¬°^Ð)JÐ)JÀrÐ)JÐ)JÑKÔKˆDŒMà9@ÐS�j�j ¤Ð5Ð5°"Ð5Ð5Ð5ÅbÄkÁmÄmˆŒØ9@ÐS�j�j ¤Ð5Ð5°"Ð5Ð5Ð5ÅbÄkÁmÄmˆŒÝœ IÑ.Ô.ˆŒåœ
 9Ñ-Ô-ˆŒØ �y‘{”{ˆŒàð 		UåœI c¨NÑ&:¸CÐVÐVÀiÐVÐSUÐVÐVˆDŒMØ!%ˆDÔØ $ˆDÔÐð !ˆDŒMÝ!#¤¨3°Ð!JÐ!J¸)Ð!JÀrÐ!JÐ!JˆDÔÝ "¤	¨.¸#Ð TÐ TÀIÐ TÐQSÐ TÐ TˆDÔàDOÐD[•*˜SÐ@Ð@¨kÐ@¸RÐ@Ð@Ð@ÕacÔalÑanÔanˆŒØ09¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒð 	×ÒÑÔÐÐÐrU   c                 ó`   — | j         �&t          j                             | j         ¦  «         dS dS )ú"Initialize parameters and buffers.N)r¦   rn   r—   Úzeros_rš   s    rS   r¹   z%ParallelScalingBlock.reset_parametersq  s.   € àŒ=Ð$ÝŒG�NŠN˜4œ=Ñ)Ô)Ð)Ð)Ð)ð %Ð$rU   r{   r|   r}   c                 óv  — |j         \  }}}|                      |¦  «        }|                      |¦  «        }t          j        || j        d¬¦  «        \  }}	}
}| j        �
|| j        z   }|                      |	                     ||| j	        | j
        ¦  «        ¦  «                             dd¦  «        }	|                      |
                     ||| j	        | j
        ¦  «        ¦  «                             dd¦  «        }
|                     ||| j	        | j
        ¦  «                             dd¦  «        }| j        r.t          j        |	|
||| j        r| j        j        nd|¬¦  «        }nv|	| j        z  }	|	|
                     dd¦  «        z  }t)          ||||¬¦  «        }t+          ||¦  «        }|                     d¬¦  «        }|                      |¦  «        }||z  }|                     dd¦  «                             |||¦  «        }|                      |¦  «        }|                      |¦  «        }| j        �,|                      t          j        ||fd¬¦  «        ¦  «        }n+|                      |¦  «        |                      |¦  «        z   }||                      |                      |¦  «        ¦  «        z   }|S )	Néÿÿÿÿ©rA   r0   é   r?   ©r|   Ú	dropout_pr}   éþÿÿÿ©r}   ) Úshaper©   r«   r�   Úsplitr¬   r¦   r°   ÚviewrB   r§   Ú	transposer±   rŸ   ÚFÚscaled_dot_product_attentionÚtrainingrG   Úpr¨   r-   r,   ÚsoftmaxÚreshaper´   r³   rµ   Úcatr¶   r·   r]   r¸   )rx   r{   r|   r}   ÚBÚNÚCÚyÚx_mlpÚqÚkÚvÚx_attnr=   Ú	attn_biass                  rS   r‚   zParallelScalingBlock.forwardv  s…  € ð ”'‰ˆˆ1ˆað �LŠL˜‰OŒOˆØ�LŠL˜‰OŒOˆÝœ Q¨¬¸2Ð>Ñ>Ô>‰ˆˆq�!�QØŒ=Ð$Ø˜DœMÑ)ˆEð �KŠK˜Ÿš˜q ! T¤^°T´]ÑCÔCÑDÔD×NÒNÈqÐRSÑTÔTˆØ�KŠK˜Ÿš˜q ! T¤^°T´]ÑCÔCÑDÔD×NÒNÈqÐRSÑTÔTˆØ�FŠF�1�a˜œ¨¬Ñ7Ô7×AÒAÀ!ÀQÑGÔGˆØŒ?ð 	ÝÔ3Ø�1�aØ#Ø.2¬mÐC˜$œ.Ô*Ð*ÀØ#ð	ñ ô ˆFˆFð �D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDÝ.¨q°$¸	ÈYÐWÑWÔWˆIÝ! $¨	Ñ2Ô2ˆDØ—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDØ˜A‘XˆFà×!Ò! ! QÑ'Ô'×/Ò/°°1°aÑ8Ô8ˆð —’˜UÑ#Ô#ˆØ—’˜eÑ$Ô$ˆð Œ=Ð$Ø—’�eœi¨°¨¸RÐ@Ñ@Ô@ÑAÔAˆAˆAà×"Ò" 6Ñ*Ô*¨T×->Ò->¸uÑ-EÔ-EÑEˆAð �—’˜tŸwšw q™zœzÑ*Ô*Ñ*ˆØˆrU   rœ   rƒ   )r…   r†   r‡   rˆ   r   r‹   Ú__annotations__rn   r‰   r   rs   rŠ   r   r   rŒ   r.   rl   r¹   r�   rŽ   r‚   r�   r�   s   @rS   rž   rž   '  s÷  ø€ € € € € € ðð ð �d”ÐÐÑð  "Ø"Ø!Ø$)Ø#(Ø"Ø!Ø!Ø+/Ø!Ø)+¬Ø*3Ø37Ø.2ØØ"'ØØð+A ð A àðA ð ðA ð ð	A ð
 ðA ð ðA ð "ðA ð !ðA ð ðA ð ðA ð ðA ð " %œðA ð ðA ð ˜BœI”ðA ð ˜RœYœðA ð     R¤Y¤Ô0ð!A ð" ! Ô+ð#A ð$ ð%A ð&  ð'A ð, 
ð-A ð A ð A ð A ð A ð A ðF*ð *ð *ð *ð 15Ø#ð	1ð 1àŒ|ð1ð   ¤Ô-ð1ð ð	1ð
 
Œð1ð 1ð 1ð 1ð 1ð 1ð 1ð 1rU   rž   c            '       ó�  ‡ — e Zd ZU dZee         ed<   ddddddddddej        e	ddddddfd	e
d
e
dededededededededee         dedeej                 deej                 deeej                          dee         de
deddf&ˆ fd„Zde
fd„Zd$d„Zdej        fd„Z	 	 d%d ej        d!eej                 d"edej        fd#„Zˆ xZS )&ÚDiffParallelScalingBlockaM   Parallel ViT block with Differential Attention (MLP & Attention in parallel).

    Combines the parallel MLP+Attention structure from 'Scaling Vision Transformers to
    22 Billion Parameters' (https://arxiv.org/abs/2302.05442) with differential attention
    from 'Differential Transformer' (https://arxiv.org/abs/2410.05258).
    rŸ   rX   FTr?   Nr   rA   rB   rY   rC   rD   rZ   r[   rF   rH   rG   r\   r]   r^   rI   r_   r@   rJ   Údual_lambdarK   c                 ó  •— t          ¦   «                              ¦   «          ||dœ}||z  dk    s
J d¦   «         ‚|s|r
J d¦   «         ‚|| _        ||z  dz  | _        | j        dz  | _        t          ¦   «         | _        t          ||z  ¦  «        }|d|z  z   } ||fi |¤Ž| _        t          j
        ||fd|i|¤Ž| _        |g|gdz  z   | _        |r|                      d	d ¦  «         n)t          j        t          j        |fi |¤Ž¦  «        | _        |r || j        fi |¤Žnt          j        ¦   «         | _        |r || j        fi |¤Žnt          j        ¦   «         | _        t          j        |
¦  «        | _        |
| _        t1          d| j        z  fd
di|¤Ž| _        || _        |rŽt          j        t          j        dt          j        |¬¦  «        ¦  «        | _        t          j        t          j        dt          j        |¬¦  «        ¦  «        | _        d x| _        x| _        x| _         | _!        �nd x| _        | _        t          j        t          j        | j        t          j        |¬¦  «        ¦  «        | _        t          j        t          j        | j        t          j        |¬¦  «        ¦  «        | _        t          j        t          j        | j        t          j        |¬¦  «        ¦  «        | _         t          j        t          j        | j        t          j        |¬¦  «        ¦  «        | _!        t          j        |	¦  «        | _"         |¦   «         | _#        t          j
        ||z   |fd|i|¤Ž| _$        |�tK          |fd|i|¤Žnt          j        ¦   «         | _&        |dk    rtO          |¦  «        nt          j        ¦   «         | _(        d| _)        |  *                    |¦  «         |  +                    ¦   «          d S )Nra   r   r¢   r£   rÂ   r¤   r¥   rh   r¦   Úepsçñhãˆµøä>rj   )rc   rb   r\   r?   çš™™™™™é?),rk   rl   rB   r§   r¨   r)   rŸ   rs   r©   rn   rª   r«   r¬   r­   r®   r�   r¯   r¦   ro   r°   r±   r²   rG   Úattn_drop_pr    Úsub_normrß   Úfloat32Úlambda_aÚlambda_bÚ	lambda_q1Ú	lambda_k1Ú	lambda_q2Ú	lambda_k2r³   r´   rµ   r/   r¸   r!   r]   Úlambda_initÚset_lambda_initr¹   )rx   rA   rB   rY   rC   rD   rZ   r[   rF   rH   rG   r\   r]   r^   rI   r_   r@   rJ   rß   rb   rc   ry   rº   r»   rz   s                           €rS   rl   z!DiffParallelScalingBlock.__init__³  sÉ  ø€ õ. 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØ�Y‰ !Ò#Ð#Ð#Ð%KÑ#Ô#Ð#Ø"ÐV¨>ÐVÐVÐ;VÑVÔVÐ9Ø"ˆŒØ˜yÑ(¨AÑ-ˆŒØ”] dÑ*ˆŒ
Ý(Ñ*Ô*ˆŒÝ˜Y¨™_Ñ-Ô-ˆØ(¨1¨s©7Ñ2ˆà!�z #Ð,Ð,¨Ð,Ð,ˆŒÝ”y  oÐKÐK¸HÐKÈÐKÐKˆŒØ'Ð(¨C¨5°1©9Ñ4ˆŒØð 	Là×#Ò# J°Ñ5Ô5Ð5Ð5åœL­¬°^Ð)JÐ)JÀrÐ)JÐ)JÑKÔKˆDŒMà9@ÐS�j�j ¤Ð5Ð5°"Ð5Ð5Ð5ÅbÄkÁmÄmˆŒØ9@ÐS�j�j ¤Ð5Ð5°"Ð5Ð5Ð5ÅbÄkÁmÄmˆŒÝœ IÑ.Ô.ˆŒØ$ˆÔõ    D¤MÑ 1ÐBÐB°tÐB¸rÐBÐBˆŒØ&ˆÔØð 		jÝœL­¬°R½u¼}ÐU[Ð)\Ñ)\Ô)\Ñ]Ô]ˆDŒMÝœL­¬°R½u¼}ÐU[Ð)\Ñ)\Ô)\Ñ]Ô]ˆDŒMØPTÐTˆDŒNÐT˜Tœ^ÐT¨d¬n¸t¼~¹~à,0Ð0ˆDŒM˜DœMÝœ\­%¬+°d´mÍ5Ì=ÐagÐ*hÑ*hÔ*hÑiÔiˆDŒNÝœ\­%¬+°d´mÍ5Ì=ÐagÐ*hÑ*hÔ*hÑiÔiˆDŒNÝœ\­%¬+°d´mÍ5Ì=ÐagÐ*hÑ*hÔ*hÑiÔiˆDŒNÝœ\­%¬+°d´mÍ5Ì=ÐagÐ*hÑ*hÔ*hÑiÔiˆDŒNåœ
 9Ñ-Ô-ˆŒØ �y‘{”{ˆŒõ œ	 #¨Ñ"6¸ÐRÐRÀ)ÐRÈrÐRÐRˆŒàDOÐD[•*˜SÐ@Ð@¨kÐ@¸RÐ@Ð@Ð@ÕacÔalÑanÔanˆŒØ09¸B²°� )Ñ,Ô,Ð,ÅBÄKÁMÄMˆŒàˆÔØ×Ò˜UÑ#Ô#Ð#ð 	×ÒÑÔÐÐÐrU   c                 óJ   — ddt          j        d|z  ¦  «        z  z
  | _        d S )Nrã   g333333ã?g333333Ó¿)ÚmathÚexprí   )rx   rJ   s     rS   rî   z(DiffParallelScalingBlock.set_lambda_init   s(   € Ø ¥t¤x°°u±Ñ'=Ô'=Ñ!=Ñ=ˆÔÐÐrU   c                 ó6  — | j         �$t          j                             | j         ¦  «         | j        rJt          j                             | j        ¦  «         t          j                             | j        ¦  «         dS t          j                             | j        dd¬¦  «         t          j                             | j	        dd¬¦  «         t          j                             | j
        dd¬¦  «         t          j                             | j        dd¬¦  «         dS )r½   Nr   çš™™™™™¹?©ÚmeanÚstd)r¦   rn   r—   r¾   rß   rç   rè   Únormal_ré   rê   rë   rì   rš   s    rS   r¹   z)DiffParallelScalingBlock.reset_parameters  sÓ   € àŒ=Ð$ÝŒG�NŠN˜4œ=Ñ)Ô)Ð)ØÔð 	=ÝŒG�NŠN˜4œ=Ñ)Ô)Ð)ÝŒG�NŠN˜4œ=Ñ)Ô)Ð)Ð)Ð)åŒG�OŠO˜DœN°¸ˆOÑ<Ô<Ð<ÝŒG�OŠO˜DœN°¸ˆOÑ<Ô<Ð<ÝŒG�OŠO˜DœN°¸ˆOÑ<Ô<Ð<ÝŒG�OŠO˜DœN°¸ˆOÑ<Ô<Ð<Ð<Ð<rU   c                 ó¬  — | j         �3t          j        | j         ¦  «        }t          j        | j        ¦  «        }nŽt          j        t          j        | j        | j        z  d¬¦  «                             ¦   «         ¦  «        }t          j        t          j        | j        | j	        z  d¬¦  «                             ¦   «         ¦  «        }||z
  | j
        z   S )NrÀ   rÁ   )rç   r�   rñ   rè   Úsumré   rê   rŠ   rë   rì   rí   )rx   Úlambda_1Úlambda_2s      rS   Ú_compute_lambdaz(DiffParallelScalingBlock._compute_lambda  s©   € ØŒ=Ð$Ý”y ¤Ñ/Ô/ˆHÝ”y ¤Ñ/Ô/ˆHˆHå”y¥¤¨4¬>¸D¼NÑ+JÐPRÐ!SÑ!SÔ!S×!YÒ!YÑ![Ô![Ñ\Ô\ˆHÝ”y¥¤¨4¬>¸D¼NÑ+JÐPRÐ!SÑ!SÔ!S×!YÒ!YÑ![Ô![Ñ\Ô\ˆHØ˜(Ñ" TÔ%5Ñ5Ð5rU   r{   r|   r}   c                 óf  — |j         \  }}}|                      |¦  «        }|                      |¦  «        }t          j        || j        d¬¦  «        \  }}	}
}| j        �
|| j        z   }|	                     ||d| j        z  | j	        ¦  «         
                    dd¦  «        }	|
                     ||d| j        z  | j	        ¦  «         
                    dd¦  «        }
|                     ||| j        d| j	        z  ¦  «         
                    dd¦  «        }|                      |	¦  «        |                      |
¦  «        }
}	|                      ¦   «                              |	¦  «        }| j        rÃ|	                     || j        d|| j	        ¦  «        }	|
                     || j        d|| j	        ¦  «        }
|	                     d¦  «        \  }}|
                     d¦  «        \  }}| j        r| j        nd}t'          j        ||||||¬¦  «        }t'          j        ||||||¬¦  «        }|||z  z
  }n¶|	| j        z  }	|	|
 
                    dd¦  «        z  }t-          ||||¬¦  «        }t/          ||¦  «        }|                     d¬¦  «        }|                      |¦  «        }|                     || j        d||¦  «        }|d d …d d …d	f         ||d d …d d …df         z  z
  }||z  }|                      |¦  «        }|d| j        z
  z  }| 
                    dd¦  «                             |||¦  «        }|                      |¦  «        }|                      |¦  «        }|                      t          j         ||fd¬¦  «        ¦  «        }||  !                    |  "                    |¦  «        ¦  «        z   }|S )
NrÀ   rÁ   rÂ   r0   r?   rÃ   rÅ   rÆ   r   )#rÇ   r©   r«   r�   rÈ   r¬   r¦   rÐ   rB   r§   rÊ   r°   r±   rü   Útype_asrŸ   ÚunbindrÍ   rä   rË   rÌ   r¨   r-   r,   rÏ   rG   rÉ   rå   rí   r´   r³   rµ   rÑ   r]   r¸   )rx   r{   r|   r}   rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   Úlambda_fullÚq1Úq2Úk1Úk2rÄ   Úattn1Úattn2rÚ   r=   rÛ   s                          rS   r‚   z DiffParallelScalingBlock.forward  s~  € ð ”'‰ˆˆ1ˆað �LŠL˜‰OŒOˆØ�LŠL˜‰OŒOˆÝœ Q¨¬¸2Ð>Ñ>Ô>‰ˆˆq�!�QØŒ=Ð$Ø˜DœMÑ)ˆEð �IŠI�a˜˜A ¤Ñ.°´Ñ>Ô>×HÒHÈÈAÑNÔNˆØ�IŠI�a˜˜A ¤Ñ.°´Ñ>Ô>×HÒHÈÈAÑNÔNˆØ�IŠI�a˜˜DœN¨A°´Ñ,=Ñ>Ô>×HÒHÈÈAÑNÔNˆà�{Š{˜1‰~Œ~˜tŸ{š{¨1™~œ~ˆ1ˆà×*Ò*Ñ,Ô,×4Ò4°QÑ7Ô7ˆàŒ?ð 	Ø—	’	˜!˜Tœ^¨Q°°4´=ÑAÔAˆAØ—	’	˜!˜Tœ^¨Q°°4´=ÑAÔAˆAØ—X’X˜a‘[”[‰FˆB�Ø—X’X˜a‘[”[‰FˆB�à,0¬MÐB˜Ô(Ð(¸sˆIÝÔ2°2°r¸1È	Ð]fÐr{Ð|Ñ|Ô|ˆEÝÔ2°2°r¸1È	Ð]fÐr{Ð|Ñ|Ô|ˆEà˜[¨5Ñ0Ñ0ˆFˆFà�D”J‘ˆAØ�q—{’{ 2 rÑ*Ô*Ñ*ˆDÝ.¨q°$¸	ÈYÐWÑWÔWˆIÝ! $¨	Ñ2Ô2ˆDØ—<’< B�<Ñ'Ô'ˆDØ—>’> $Ñ'Ô'ˆDà—9’9˜Q ¤°°1°aÑ8Ô8ˆDØ˜˜˜˜1˜1˜1˜a˜”= ;°°a°a°a¸¸¸¸A°g´Ñ#>Ñ>ˆDØ˜A‘XˆFà—’˜vÑ&Ô&ˆØ˜1˜tÔ/Ñ/Ñ0ˆØ×!Ò! ! QÑ'Ô'×/Ò/°°1°aÑ8Ô8ˆð —’˜UÑ#Ô#ˆØ—’˜eÑ$Ô$ˆð �MŠM�%œ) V¨U O¸Ð<Ñ<Ô<Ñ=Ô=ˆð �—’˜tŸwšw q™zœzÑ*Ô*Ñ*ˆØˆrU   rœ   rƒ   )r…   r†   r‡   rˆ   r   r‹   rÜ   rn   r‰   r   rs   rŠ   r   r   rŒ   r.   rl   rî   r¹   r�   rŽ   rü   r‚   r�   r�   s   @rS   rÞ   rÞ   ª  s/  ø€ € € € € € ðð ð �d”ÐÐÑð  "Ø"Ø!Ø$)Ø#(Ø"Ø!Ø!Ø+/Ø!Ø)+¬Ø*3Ø37Ø.2ØØ %ØØð+K ð K àðK ð ðK ð ð	K ð
 ðK ð ðK ð "ðK ð !ðK ð ðK ð ðK ð ðK ð " %œðK ð ðK ð ˜BœI”ðK ð ˜RœYœðK ð     R¤Y¤Ô0ð!K ð" ! Ô+ð#K ð$ ð%K ð& ð'K ð, 
ð-K ð K ð K ð K ð K ð K ðZ> Sð >ð >ð >ð >ð=ð =ð =ð =ð6 ¤ð 6ð 6ð 6ð 6ð 15Ø#ð	<ð <àŒ|ð<ð   ¤Ô-ð<ð ð	<ð
 
Œð<ð <ð <ð <ð <ð <ð <ð <rU   rÞ   c            '       ó6  ‡ — e Zd ZdZdddddddddddej        eeedddfd	e	d
e	de	de
dedededededee
         de
de
de
deej                 deej                 deej                 dede	ddf&ˆ fd„Z	 	 d!dej        deej                 dedej        fd „Zˆ xZS )"ÚParallelThingsBlockz¿ Parallel ViT block (N parallel attention followed by N parallel MLP)
    Based on:
      `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    rÂ   rX   FTNr?   r   rA   rB   Únum_parallelrY   rC   rD   rZ   r[   rF   r\   rH   rG   r]   r^   rI   r_   r@   rJ   rK   c                 ó¦  •— ||dœ}t          ¦   «                              ¦   «          || _        t          j        ¦   «         | _        t          j        ¦   «         | _        t          |¦  «        D �]b}| j                             t          j	        t          d ||fi |¤Žfdt          ||f|||||	||||dœ	|¤Žfd|
rt          |fd|
i|¤Žnt          j        ¦   «         fd|dk    rt          |¦  «        nt          j        ¦   «         fg¦  «        ¦  «        ¦  «         | j                             t          j	        t          d ||fi |¤Žfd	 ||ft          ||z  ¦  «        ||r|nd |	|d
œ|¤Žfd|
rt          |fd|
i|¤Žnt          j        ¦   «         fd|dk    rt          |¦  «        nt          j        ¦   «         fg¦  «        ¦  «        ¦  «         �Œdd S )Nra   Únormr=   rd   r¸   r\   r]   r?   rt   )rg   r^   rI   rh   ri   )rk   rl   r	  rn   Ú
ModuleListÚattnsÚffnsÚrangeÚappendÚ
Sequentialr   rT   r/   ro   r!   rs   )rx   rA   rB   r	  rY   rC   rD   rZ   r[   rF   r\   rH   rG   r]   r^   rI   r_   r@   rJ   rb   rc   ry   Ú_rz   s                          €rS   rl   zParallelThingsBlock.__init__]  s`  ø€ ð. ¨Ð/Ð/ˆÝ‰Œ×ÒÑÔÐØ(ˆÔÝ”]‘_”_ˆŒ
Ý”M‘O”OˆŒ	Ý�|Ñ$Ô$ð !	ñ !	ˆAØŒJ×Ò�bœm­KØ˜˜ CÐ.Ð.¨2Ð.Ð.Ð/Ø�ØØðð (Ø%Ø#Ø.Ø'Ø'Ø'Ø)Øðð ð ðð ð ð ÈÐg•z #ÐEÐE°;ÐEÀ"ÐEÐEÐEÕZ\ÔZeÑZgÔZgÐhØ°YÀ²^°^�h yÑ1Ô1Ð1ÍÌÉÌÐWð#9ñ -ô -ñ ô ñ ô ð ð& ŒI×Ò�Rœ]­;Ø˜˜ CÐ.Ð.¨2Ð.Ð.Ð/Ø˜	˜	Øðå$'¨¨i©Ñ$8Ô$8Ø'Ø-;ÐE˜z˜zÀØ"Ø"ðð ð ðð ð ð ÈÐg•z #ÐEÐE°;ÐEÀ"ÐEÐEÐEÕZ\ÔZeÑZgÔZgÐhØ°YÀ²^°^�h yÑ1Ô1Ð1ÍÌÉÌÐWð8ñ ,ô ,ñ ô ñ ô ð ñ ð)!	ð !	rU   r{   r|   r}   c                 óH  ‡— |€|r¤g }| j         D ]n}|                     ‰¦  «        }|                     |||¬¦  «        }|                     |¦  «        }|                     |¦  «        }|                     |¦  «         Œo‰t          j        |¦  «                             d¬¦  «        z   Šn<‰t          j        ˆfd„| j         D ¦   «         ¦  «                             d¬¦  «        z   Š‰t          j        ˆfd„| j	        D ¦   «         ¦  «                             d¬¦  «        z   Š‰S )Nr€   r   rÁ   c                 ó&   •— g | ]} |‰¦  «        ‘ŒS rj   rj   )Ú.0r=   r{   s     €rS   ú
<listcomp>z/ParallelThingsBlock.forward.<locals>.<listcomp>¬  s!   ø€ Ð @Ð @Ð @¨T   a¡¤Ð @Ð @Ð @rU   c                 ó&   •— g | ]} |‰¦  «        ‘ŒS rj   rj   )r  Úffnr{   s     €rS   r  z/ParallelThingsBlock.forward.<locals>.<listcomp>­  s!   ø€ Ð9Ð9Ð9¨˜S˜S ™VœVÐ9Ð9Ð9rU   )
r  r  r=   r¸   r]   r  r�   Ústackrù   r  )rx   r{   r|   r}   Úattn_outr=   rÚ   s    `     rS   r‚   zParallelThingsBlock.forwardœ  s%  ø€ ð Ð  IÐ ØˆHØœ
ð (ð (�ØŸš 1™œ�ØŸš 6°YÈ)˜ÑTÔT�ØŸš ™œ�ØŸš¨Ñ/Ô/�Ø—’ Ñ'Ô'Ð'Ð'Ø•E”K Ñ)Ô)×-Ò-°!Ð-Ñ4Ô4Ñ4ˆAˆAà•E”KÐ @Ð @Ð @Ð @°T´ZÐ @Ñ @Ô @ÑAÔA×EÒEÈ!ÐEÑLÔLÑLˆAØ•”Ð9Ð9Ð9Ð9¨t¬yÐ9Ñ9Ô9Ñ:Ô:×>Ò>À1Ð>ÑEÔEÑEˆØˆrU   rƒ   r„   r�   s   @rS   r  r  X  s«  ø€ € € € € ðð ð !"Ø!Ø"Ø!Ø$)Ø#(Ø"Ø+/Ø!Ø!Ø!Ø)+¬Ø*3Ø),Ø$-ØØØð+=ð =àð=ð ð=ð ð	=ð
 ð=ð ð=ð ð=ð "ð=ð !ð=ð ð=ð " %œð=ð ð=ð ð=ð ð=ð ˜BœI”ð=ð  ˜RœYœð!=ð" ˜BœI”ð#=ð$ "ð%=ð& ð'=ð, 
ð-=ð =ð =ð =ð =ð =ðD 15Ø#ð	ð àŒ|ðð   ¤Ô-ðð ð	ð
 
Œðð ð ð ð ð ð ð rU   r  Útokenr{   Ú	pool_typeÚnum_prefix_tokensÚreduce_include_prefixc                 óX  — |s| S |dk    r| d d …df         } n’|r| n| d d …|d …f         } |dk    r|                       d¬¦  «        } nc|dk    r1d|                      d¬¦  «        |                       d¬¦  «        z   z  } n,|dk    r|                      d¬¦  «        } n|rJ d	|› �¦   «         ‚| S )
Nr  r   Úavgr0   rÁ   Úavgmaxg      à?ÚmaxzUnknown pool type )rõ   Úamax)r{   r  r  r  s       rS   Úglobal_pool_nlcr$  ±  sã   € ð ð Øˆà�GÒÐØˆaˆaˆa�ˆdŒGˆˆà&ÐDˆAˆA¨A¨a¨a¨aÐ1BÐ1CÐ1CÐ.CÔ,DˆØ˜ÒÐØ—’˜1�‘”ˆAˆAØ˜(Ò"Ð"Ø�q—v’v !�v‘}”} q§v¢v°! v¡}¤}Ñ4Ñ5ˆAˆAØ˜%ÒÐØ—’˜1�‘”ˆAˆAà ÐBÐBÐ"B°yÐ"BÐ"BÑBÔB�=à€HrU   c            S       óÜ  ‡ — e Zd ZU dZee         ed<   ddddddd	d	d
ddddddddddddddddddddddddedddee	e
ddf*deeeeef         f         deeeeef         f         dededed         dedededededededed ed!ee         d"ed#ed$ed%ed&ed'ed(ee         d)eded*ed+ed,ed-ed.ed/ed0ed1ed2         d3ed4ed5ee         d6ee         d7ee         d8eej                 d9eej                 d:ed;dfRˆ fd<„Zdhd=„Zdid>ed?ed;dfd@„ZdAej        d;dfdB„Zej                             ¦   «         djdCedDed;dfdE„¦   «         Zej        j        d;ee         fdF„¦   «         Z ej        j        dkdGed;e!eeee"f         f         fdH„¦   «         Z#ej        j        dldIed;dfdJ„¦   «         Z$ej        j        d;ej        fdK„¦   «         Z%dmdedee         d;dfdL„Z&	 	 dndeeeef                  deeeef                  d;dfdM„Z'dNej(        d;ej(        fdO„Z)	 	 	 	 	 	 	 	 	 dodNej(        dQeeee"e         f                  dRedSedTedUedVedWedXeej(                 dYed;ee"ej(                 eej(        e"ej(                 f         e!ee*f         f         fdZ„Z+	 	 	 dpdQeee"e         f         d\ed]ed;e"e         fd^„Z,	 	 	 	 	 dqdNej(        d_eee"e         ee         f         d`edRedSedXeej(                 d;e"ej(                 fda„Z-	 	 drdNej(        dXeej(                 dYed;ej(        fdb„Z.dmdNej(        dcee         d;ej(        fdd„Z/dkdNej(        deed;ej(        fdf„Z0	 	 drdNej(        dXeej(                 dYed;ej(        fdg„Z1ˆ xZ2S )sr;   z© Vision Transformer

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929
    Údynamic_img_sizeéà   é   r¥   éè  r  é   é   rX   TFNÚlearnr   r?   r<   Úimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_pool©r<   r   r!  r"  r  ÚmapÚprrÚ	embed_dimrJ   rB   rY   rC   rD   rZ   r[   rF   r\   Úclass_tokenÚ	pos_embedÚno_embed_classÚ
reg_tokensÚpre_normÚ
final_normÚfc_normÚpool_include_prefixÚdynamic_img_padÚ	drop_rateÚpos_drop_rateÚpatch_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚweight_init)ÚskipÚresetÚjaxÚjax_nlhbÚmocor<   Úfix_initÚembed_layerÚembed_norm_layerrI   r^   Úblock_fnr_   r@   rK   c+                 óv  •‡‡‡	‡
‡‡‡‡‡‡‡‡$‡%‡&‡'‡(‡/‡0‡1— t          ¦   «                              ¦   «          |)|*dœŠ/|dv sJ ‚|s|dk    sJ ‚|dv sJ ‚|€|dv n|}+t          ‰$¦  «        pt          Š$t          |#¦  «        }#t	          ‰%¦  «        pt
          j        Š%|| _        || _        || _	        ‰x| _
        x| _        | _        |rdnd| _        | xj        |z  c_        || _        || _        || _        || _        || _        d	| _        i },|r$|,                     t+          d	d
¬¦  «        ¦  «         |#�|#|,d<    |"d|||‰| |dœ|,¤‰/¤Ž| _        | j        j        }-t1          | j        d¦  «        r| j                             ¦   «         n|Š1|r&t          j        t7          j        dd‰fi ‰/¤Ž¦  «        nd| _        |r&t          j        t7          j        d|‰fi ‰/¤Ž¦  «        nd| _        |r|-n	|-| j        z   }.|r|dk    rd| _        n+t          j        t7          j        d|.‰fi ‰/¤Ž¦  «        | _        t          j         |¬¦  «        | _!        |dk    rtE          || j        ¬¦  «        | _#        nt          j$        ¦   «         | _#        |r	 ‰$‰fi ‰/¤Žnt          j$        ¦   «         | _%        tM          ||¦  «        Š0t          j'        ˆ%ˆˆ(ˆ&ˆ/ˆ0ˆˆˆ'ˆ	ˆ$ˆˆˆˆˆ
ˆˆfd„tQ          |¦  «        D ¦   «         Ž | _)        ˆˆ1fd„tQ          |¦  «        D ¦   «         | _*        |r|+s	 ‰$‰fi ‰/¤Žnt          j$        ¦   «         | _+        |dk    rtY          | j        f‰‰	‰$‰%dœ‰/¤Ž| _-        n4|dk    r't]          | j        f‰|rdnd‰$dœ‰/¤Ž| _-        d| _        nd| _-        |r|+r	 ‰$‰fi ‰/¤Žnt          j$        ¦   «         | _/        t          j         |¦  «        | _0        |dk    rt          j1        | j        |fi ‰/¤Žnt          j$        ¦   «         | _2        | dk    rdn| | _3        |!| _4        | dk    r|  5                    d	¬¦  «         dS dS )aò  
        Args:
            img_size: Input image size.
            patch_size: Patch size.
            in_chans: Number of image input channels.
            num_classes: Number of classes for classification head.
            global_pool: Type of global pooling for final sequence (default: 'token').
            embed_dim: Transformer embedding dimension.
            depth: Depth of transformer.
            num_heads: Number of attention heads.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: Enable bias for qkv projections if True.
            init_values: Layer-scale init values (layer-scale enabled if not None).
            class_token: Use class token.
            no_embed_class: Don't include position embeddings for class (or reg) tokens.
            reg_tokens: Number of register tokens.
            pre_norm: Enable norm after embeddings, before transformer blocks (standard in CLIP ViT).
            final_norm: Enable norm after transformer blocks, before head (standard in most ViT).
            fc_norm: Move final norm after pool (instead of before), if None, enabled when global_pool == 'avg'.
            drop_rate: Head dropout rate.
            pos_drop_rate: Position embedding dropout rate.
            attn_drop_rate: Attention dropout rate.
            drop_path_rate: Stochastic depth rate.
            weight_init: Weight initialization scheme.
            fix_init: Apply weight initialization fix (scaling w/ layer index).
            embed_layer: Patch embedding layer.
            embed_norm_layer: Normalization layer to use / override in patch embed module.
            norm_layer: Normalization layer.
            act_layer: MLP activation layer.
            block_fn: Transformer block layer.
        ra   r2  r  )r<   Únoner,  N)r   r!  r"  r0   r   FÚNHWC)Ústrict_img_sizeÚ
output_fmtrI   )r-  r.  r/  r5  rh   r>  Ú
feat_ratiorP  )rÎ   ©r  c           	      ó’   •— g | ]C} ‰di d ‰“d‰“d‰“d‰“d‰“d‰“d‰“d‰“d‰	“d	‰“d
‰“d‰|         “d‰“d‰“d‰
“d‰“d|“‰¤Ž‘ŒDS )rA   rB   rY   rC   rD   rZ   r[   rF   r\   rH   rG   r]   rI   r^   r_   r@   rJ   rj   rj   )r  Úir^   rC  r@   rN  ry   Údprr5  r\   r_   rY   rI   rB   rF   rB  rD   rC   rZ   r[   s     €€€€€€€€€€€€€€€€€€rS   r  z.VisionTransformer.__init__.<locals>.<listcomp>Z  s  ø€ ð &#ð &#ð &#ð* ð) ˆHð ð ð Ø�Iðà#˜)ðð $˜)ðð "˜ð	ð
  ˜ðð !0 ðð  .˜~ðð $˜)ðð (˜Kðð )˜.ðð )˜.ðð ˜aœ&˜&ðð &˜:ðð $˜)ðð $˜)ðð  &˜:ð!ð" �aØð%ð ð&#ð &#ð &#rU   c                 ó:   •— g | ]}t          d |› �‰‰¬¦  «        ‘ŒS )úblocks.)ÚmoduleÚnum_chsÚ	reduction©Údict)r  rW  r5  r]  s     €€rS   r  z.VisionTransformer.__init__.<locals>.<listcomp>p  sA   ø€ ð fð fð fØST�D˜ !˜˜¨yÀIÐNÑNÔNðfð fð frU   r3  )rB   rY   rI   r^   r4  r   )rB   r  rI   TrF  rG  ©Úneeds_resetrj   )6rk   rl   r+   r   r*   rn   r‰   r0  r/  r1  Únum_featuresÚhead_hidden_sizer5  r  Únum_reg_tokensÚhas_class_tokenr8  r=  r&  Úgrad_checkpointingÚupdater_  Úpatch_embedÚnum_patchesÚhasattrrT  r®   r�   r¯   Ú	cls_tokenÚ	reg_tokenr7  r²   Úpos_dropr#   Ú
patch_dropro   Únorm_prer"   r  r  ÚblocksÚfeature_infor  r   Ú	attn_poolr   r<  Ú	head_droprª   ÚheadÚweight_init_moderK  r”   )3rx   r-  r.  r/  r0  r1  r5  rJ   rB   rY   rC   rD   rZ   r[   rF   r\   r6  r7  r8  r9  r:  r;  r<  r=  r&  r>  r?  r@  rA  rB  rC  rD  rE  rK  rL  rM  rI   r^   rN  r_   r@   rb   rc   Úuse_fc_normÚ
embed_argsri  Ú	embed_lenry   rX  r]  rz   s3         ` ````````             ``     `````      @@@€rS   rl   zVisionTransformer.__init__Ò  s±  øøøøøøøøøøøøøøøøøøøø€ õX 	‰Œ×ÒÑÔÐØ¨Ð/Ð/ˆØÐQÐQÐQÐQÐQØÐ4˜k¨WÒ4Ð4Ð4Ð4ØÐ1Ð1Ð1Ð1Ð1ØAHÀ�kÐ%=Ð=Ð=ÐV]ˆÝ# JÑ/Ô/Ð<µ9ˆ
Ý)Ð*:Ñ;Ô;ÐÝ! )Ñ,Ô,Ð7µ´ˆ	à&ˆÔØ ˆŒØ&ˆÔØENÐNˆÔÐN˜DÔ1°D´NØ&1Ð!8  °qˆÔØÐÔ *Ñ,ÐÔØ(ˆÔØ*ˆÔØ,ˆÔØ#6ˆÔ Ø 0ˆÔØ"'ˆÔàˆ
Øð 	Nà×Ò�d°5ÀVÐLÑLÔLÑMÔMÐMØÐ'Ø'7ˆJ�|Ñ$Ø&˜;ð 	
ØØ!ØØØ�Ø+ð	
ð 	
ð ð	
ð ð	
ð 	
ˆÔð Ô&Ô2ˆÝ5<¸TÔ=MÈ|Ñ5\Ô5\Ðl�DÔ$×/Ò/Ñ1Ô1Ð1Ðblˆ	àMXÐb�œ¥e¤k°!°Q¸	Ð&HÐ&HÀRÐ&HÐ&HÑIÔIÐIÐ^bˆŒØV`Ðj�œ¥e¤k°!°ZÀÐ&QÐ&QÈbÐ&QÐ&QÑRÔRÐRÐfjˆŒØ#1Ð[�K�K°{ÀTÔE[Ñ7[ˆ	Øð 	V˜I¨Ò/Ð/Ø!ˆDŒNˆNåœ\­%¬+°a¸ÀIÐ*TÐ*TÐQSÐ*TÐ*TÑUÔUˆDŒNÝœ
 ]Ð3Ñ3Ô3ˆŒØ˜QÒÐÝ*ØØ"&Ô"8ðñ ô ˆDŒOˆOõ
 !œk™mœmˆDŒOØ7?ÐR˜
˜
 9Ð3Ð3°Ð3Ð3Ð3ÅRÄ[Á]Ä]ˆŒå'¨¸Ñ>Ô>ˆÝ”mð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#ð &#õ* ˜5‘\”\ð+&#ñ &#ô &#ð $ˆŒð,fð fð fð fð fÝX]Ð^cÑXdÔXdðfñ fô fˆÔà3=ÐdÀkÐd�J�J˜yÐ/Ð/¨BÐ/Ð/Ð/ÕWYÔWbÑWdÔWdˆŒ	ð ˜%ÒÐÝ0Ø”ðà#Ø#Ø%Ø#ðð ð ðð ˆDŒNˆNð ˜EÒ!Ð!Ý-Ø”ðà#Ø%0Ð;˜'˜'°eØ%ð	ð ð
 ðð ˆDŒNð (,ˆDÔ$Ð$à!ˆDŒNØ6@ÐcÀ[Ðc�z�z )Ð2Ð2¨rÐ2Ð2Ð2ÕVXÔVaÑVcÔVcˆŒÝœ IÑ.Ô.ˆŒØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	à+6¸&Ò+@Ð+@  ÀkˆÔØ ˆŒà˜&Ò Ð Ø×Ò¨%ÐÑ0Ô0Ð0Ð0Ð0ð !Ð rU   c                 óX  — t          j        ¦   «         5  t          | j        ¦  «        D ]g\  }}t	          j        d|dz   z  ¦  «        }|j        j        j         	                    |¦  «         |j
        j        j         	                    |¦  «         Œh	 ddd¦  «         dS # 1 swxY w Y   dS )z9Apply weight initialization fix (scaling w/ layer index).g       @r0   N)r�   Úno_gradÚ	enumeraterp  rð   Úsqrtr=   Úprojr™   Údiv_rt   Úfc2)rx   Úlayer_idÚlayerr¨   s       rS   Úfix_init_weightz!VisionTransformer.fix_init_weight“  så   € åŒ]‰_Œ_ð 	1ð 	1Ý#,¨T¬[Ñ#9Ô#9ð 1ð 1‘�˜%Ýœ	 #¨°A©Ñ"6Ñ7Ô7�Ø”
”Ô&×+Ò+¨EÑ2Ô2Ð2Ø”	”Ô$×)Ò)¨%Ñ0Ô0Ð0Ð0ð1ð	1ð 	1ð 	1ñ 	1ô 	1ð 	1ð 	1ð 	1ð 	1ð 	1ð 	1ð 	1øøøð 	1ð 	1ð 	1ð 	1ð 	1ð 	1s   ”A=BÂB#Â&B#Úmodera  c                 óÌ  — |p| j         }|dv sJ ‚d|v rt          j        | j        ¦  «         nd}| j        �t          | j        d¬¦  «         | j        �&t          j         	                    | j        d¬¦  «         | j
        �&t          j         	                    | j
        d¬¦  «         t          t          |||¬¦  «        | ¦  «         | j        r|                      ¦   «          dS dS )	a*  Initialize model weights.

        Args:
            mode: Weight initialization mode ('jax', 'jax_nlhb', 'moco', or '').
            needs_reset: If True, call reset_parameters() on modules that have it.
                Set to False when modules have already self-initialized in __init__.
        )rH  rI  rJ  rG  r<   Únlhbr?   Nç{®Gáz”?©rö   ç�íµ ÷Æ°>r`  )ru  rð   Úlogr0  r7  r$   rk  rn   r—   r÷   rl  r4   Úget_init_weights_vitrK  r‚  )rx   rƒ  ra  Ú	head_biass       rS   r”   zVisionTransformer.init_weights›  só   € ð Ð,�tÔ,ˆØÐ?Ð?Ð?Ð?Ð?Ø39¸T°>°>•T”X˜dÔ.Ñ/Ô/Ð/Ð/Àrˆ	ØŒ>Ð%Ý˜$œ.¨cÐ2Ñ2Ô2Ð2ØŒ>Ð%ÝŒG�OŠO˜DœN°ˆOÑ5Ô5Ð5ØŒ>Ð%ÝŒG�OŠO˜DœN°ˆOÑ5Ô5Ð5åÕ(¨¨yÀkÐRÑRÔRÐTXÑYÔYÐYàŒ=ð 	#Ø× Ò Ñ"Ô"Ð"Ð"Ð"ð	#ð 	#rU   Úmc                 ó$   — t          |¦  «         dS )z>Initialize weights for a single module (compatibility method).N)Úinit_weights_vit_timm)rx   rŒ  s     rS   Ú_init_weightszVisionTransformer._init_weights²  s   € õ 	˜aÑ Ô Ð Ð Ð rU   Úcheckpoint_pathÚprefixc                 ó(   — t          | ||¦  «         dS )a|  Load model-specific weights from an original JAX/Flax NumPy checkpoint.

        This helper handles foreign ``.npz`` checkpoint layouts. Native PyTorch
        state dictionaries are loaded by the generic factory/checkpoint helpers.

        Args:
            checkpoint_path: Path to a NumPy checkpoint.
            prefix: Prefix for parameter names in the checkpoint.
        N)Ú_load_weights)rx   r�  r‘  s      rS   Úload_pretrainedz!VisionTransformer.load_pretrained·  s   € õ 	�d˜O¨VÑ4Ô4Ð4Ð4Ð4rU   c                 ó
   — h d£S )z3Set of parameters that should not use weight decay.>   rk  r7  rl  Ú
dist_tokenrj   rš   s    rS   Úno_weight_decayz!VisionTransformer.no_weight_decayÄ  s   € ð EÐDÐDÐDrU   Úcoarsec                 ó(   — t          dddg¬¦  «        S )z¾Create regex patterns for parameter grouping.

        Args:
            coarse: Use coarse grouping.

        Returns:
            Dictionary mapping group names to regex patterns.
        z*^cls_token|reg_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemrp  r^  )rx   r˜  s     rS   Úgroup_matcherzVisionTransformer.group_matcherÉ  s'   € õ Ø>Ø-Ð/CÐDð
ñ 
ô 
ð 	
rU   Úenablec                 óv   — || _         t          | j        d¦  «        r| j                             |¦  «         dS dS )zEnable or disable gradient checkpointing.

        Args:
            enable: Whether to enable gradient checkpointing.
        Úset_grad_checkpointingN)rf  rj  rh  rž  )rx   rœ  s     rS   rž  z(VisionTransformer.set_grad_checkpointingØ  sI   € ð #)ˆÔÝ�4Ô#Ð%=Ñ>Ô>ð 	<ØÔ×3Ò3°FÑ;Ô;Ð;Ð;Ð;ð	<ð 	<rU   c                 ó   — | j         S )zGet the classifier head.)rt  rš   s    rS   Úget_classifierz VisionTransformer.get_classifierã  s   € ð ŒyÐrU   c                 ó|  — t          | ¦  «        }|| _        |�N|dv sJ ‚|dv r| j        €
J d¦   «         ‚|dvr| j        �d| _        n|dv r| j        |k    r
J d¦   «         ‚|| _        |dk    rt	          j        | j        |fi |¤Žnt	          j        ¦   «         | _        | j         	                    | j
        ¦  «         dS )zŸReset the classifier head.

        Args:
            num_classes: Number of classes for new classifier.
            global_pool: Global pooling type.
        Nr2  )r3  r4  Fz=Cannot currently add attention pooling in reset_classifier().zECannot currently change attention pooling type in reset_classifier().r   )r   r0  rr  r1  rn   rª   r5  ro   rt  ÚtrainrÍ   )rx   r0  r1  ry   s       rS   Úreset_classifierz"VisionTransformer.reset_classifierè  së   € õ ˜dÑ#Ô#ˆØ&ˆÔØÐ"ØÐ"UÐUÐUÐUÐUØ˜nÐ,Ð,°´Ð1GØ]Ð]Ñ]Ô]�uØ NÐ2Ð2°t´~Ð7QØ!%�”�Ø Ð.Ð.°4Ô3CÀ{Ò3RÐ3RØeÐeÑeÔe�uØ*ˆDÔØDOÐRSÂOÀO•B”I˜dœn¨kÐ@Ð@¸RÐ@Ð@Ð@ÕY[ÔYdÑYfÔYfˆŒ	ØŒ	�Š˜œÑ&Ô&Ð&Ð&Ð&rU   c           	      óJ  — | j         j        }| j                              ||¬¦  «         | j        �q| j        rdn| j        }| j         j        |z   }|| j        j        d         k    r>t          j	        t          | j        | j         j        ||d¬¦  «        ¦  «        | _        dS dS dS )zæUpdate the input image resolution and patch size.

        Args:
            img_size: New input resolution, if None current resolution is used.
            patch_size: New patch size, if None existing patch size is used.
        )r-  r.  Nr   r0   T)Únew_sizeÚold_sizer  Úverbose)rh  Ú	grid_sizeÚset_input_sizer7  r8  r  ri  rÇ   rn   r®   r'   )rx   r-  r.  Úprev_grid_sizer  Únum_new_tokenss         rS   r©  z VisionTransformer.set_input_sizeý  s¾   € ð Ô)Ô3ˆØÔ×'Ò'°ÀjÐ'ÑQÔQÐQØŒ>Ð%Ø%)Ô%8Ð T  ¸dÔ>TÐØ!Ô-Ô9Ð<MÑMˆNØ ¤Ô!5°aÔ!8Ò8Ð8Ý!#¤Õ.DØ”NØ!Ô-Ô7Ø+Ø&7Ø ð/ñ /ô /ñ "ô "�”��ð	 &Ð%ð 9Ð8rU   r{   c           	      ó,  — g }| j         �:|                     | j                              |j        d         dd¦  «        ¦  «         | j        �:|                     | j                             |j        d         dd¦  «        ¦  «         | j        €Et          j        ||                     |j        d         d|j        d         ¦  «        gz   d¬¦  «        S | j	        rX|j        \  }}}}| j
        j        }t          | j        ||f|| j        rdn| j        ¬¦  «        }|                     |d|¦  «        }n| j        }| j        r"||z   }|rt          j        ||gz   d¬¦  «        }n!|rt          j        ||gz   d¬¦  «        }||z   }|                      |¦  «        S )z$Apply positional embedding to input.Nr   rÀ   r0   rÁ   )r¥  r¦  r  )rk  r  ÚexpandrÇ   rl  r7  r�   rÑ   rÉ   r&  rh  r¨  r'   r8  r  rm  )	rx   r{   Úto_catrÒ   ÚHÚWrÔ   rª  r7  s	            rS   Ú
_pos_embedzVisionTransformer._pos_embed  s£  € àˆØŒ>Ð%Ø�MŠM˜$œ.×/Ò/°´¸´
¸BÀÑCÔCÑDÔDÐDØŒ>Ð%Ø�MŠM˜$œ.×/Ò/°´¸´
¸BÀÑCÔCÑDÔDÐDàŒ>Ð!Ý”9˜V q§v¢v¨a¬g°a¬j¸"¸a¼gÀb¼kÑ'JÔ'JÐ&KÑKÐQRÐSÑSÔSÐSàÔ ð 	'Øœ‰JˆAˆq�!�QØ!Ô-Ô7ˆNÝ.Ø”Ø˜Q˜Ø'Ø'+Ô':Ð"V ! !ÀÔ@Vð	ñ ô ˆIð —’�q˜"˜aÑ Ô ˆAˆAàœˆIàÔð 	ð �I‘ˆAØð 3Ý”I˜f¨ s™l°Ð2Ñ2Ô2�øð ð 3Ý”I˜f¨ s™l°Ð2Ñ2Ô2�Ø�I‘ˆAà�}Š}˜QÑÔÐrU   ÚNCHWÚindicesÚreturn_prefix_tokensr  Ú
stop_earlyrS  Úintermediates_onlyÚoutput_dictr|   r}   c                 ó¬  ‡ ‡‡‡— |dv s
J d¦   «         ‚|dk    }g }t          t          ‰ j        ¦  «        |¦  «        \  }}|j        \  Š}}}‰                      |¦  «        }‰                      |¦  «        }‰                      |¦  «        }‰                      |¦  «        }t          j	         
                    ¦   «         s|s‰ j        }n‰ j        d|dz   …         }t          |¦  «        D ]‰\  }}|	€|
r |||	|
¬¦  «        }nA‰ j        r/t          j	         
                    ¦   «         st          ||¦  «        }n ||¦  «        }||v r,|                     |r‰                      |¦  «        n|¦  «         ŒŠ‰ j        rˆ fd„|D ¦   «         }ˆ fd„|D ¦   «         }nd}|r/‰ j                             ||f¦  «        \  ŠŠˆˆˆfd	„|D ¦   «         }|r.i }||d
<   |�|r||d<   |s‰                      |¦  «        }||d<   |S t          j	         
                    ¦   «         s!|r|�t%          t'          ||¦  «        ¦  «        }|r|S ‰                      |¦  «        }||fS )a,   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
            output_dict: Return outputs as a dictionary with 'image_features' and 'image_intermediates' keys
            attn_mask: Optional attention mask for masked attention (e.g., for NaFlex)
            is_causal: If True, use causal (autoregressive) masking in attention
        Returns:
            A tuple with (final_features, intermediates), a list of intermediate features, or a dictionary containing
            'image_features' and 'image_intermediates' (and optionally 'image_intermediates_prefix')
        )r²  ÚNLCz)Output format must be one of NCHW or NLC.r²  Nr0   r€   c                 ó6   •— g | ]}|d d …d‰j         …f         ‘ŒS )Nr   rU  ©r  rÕ   rx   s     €rS   r  z;VisionTransformer.forward_intermediates.<locals>.<listcomp>y  s/   ø€ ÐSÐSÐSÀ˜Q˜q˜q˜q ! DÔ$:Ð":Ð:Ô;ÐSÐSÐSrU   c                 ó6   •— g | ]}|d d …‰j         d …f         ‘ŒS r–   rU  r»  s     €rS   r  z;VisionTransformer.forward_intermediates.<locals>.<listcomp>z  s0   ø€ ÐRÐRÐR¸q˜Q˜q˜q˜q $Ô"8Ð"9Ð"9Ð9Ô:ÐRÐRÐRrU   c                 ó�   •— g | ]B}|                      ‰‰‰d ¦  «                             dddd¦  «                             ¦   «         ‘ŒCS )rÀ   r   r¥   r0   rÂ   )rÐ   ÚpermuteÚ
contiguous)r  rÕ   rÒ   r¯  r°  s     €€€rS   r  z;VisionTransformer.forward_intermediates.<locals>.<listcomp>�  sL   ø€ ÐlÐlÐlÐYZ˜QŸYšY q¨!¨Q°Ñ3Ô3×;Ò;¸A¸qÀ!ÀQÑGÔG×RÒRÑTÔTÐlÐlÐlrU   Úimage_intermediatesÚimage_intermediates_prefixÚimage_features)r2   Úlenrp  rÇ   rh  r±  rn  ro  r�   ÚjitÚis_scriptingr{  rf  r5   r  r  r  Údynamic_feat_sizeÚlistÚzip)rx   r{   r³  r´  r  rµ  rS  r¶  r·  r|   r}   rÐ   ÚintermediatesÚtake_indicesÚ	max_indexr  ÚheightÚwidthrp  rW  ÚblkÚprefix_tokensÚresult_dictÚx_finalrÒ   r¯  r°  s   `                       @@@rS   Úforward_intermediatesz'VisionTransformer.forward_intermediates=  sà  øøøø€ ð< ˜_Ð,Ð,Ð,Ð.YÑ,Ô,Ð,Ø Ò&ˆØˆÝ"6µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�ið  œgÑˆˆ1ˆf�eØ×Ò˜QÑÔˆØ�OŠO˜AÑÔˆØ�OŠO˜AÑÔˆØ�MŠM˜!ÑÔˆåŒ9×!Ò!Ñ#Ô#ð 	1¨:ð 	1Ø”[ˆFˆFà”[  )¨a¡- Ô0ˆFÝ Ñ'Ô'ð 		Bð 		B‰FˆAˆsØÐ$¨	Ð$Ø�C˜ Y¸)ÐDÑDÔD��ØÔ(ð µ´×1GÒ1GÑ1IÔ1Ið Ý˜s AÑ&Ô&��à�C˜‘F”F�Ø�LÐ Ð à×$Ò$°TÐ%@ T§Y¢Y¨q¡\¤\ \¸qÑAÔAÐAøð Ô!ð 	!àSÐSÐSÐSÀ]ÐSÑSÔSˆMØRÐRÐRÐRÀMÐRÑRÔRˆMˆMà ˆMàð 	màÔ#×5Ò5°v¸u°oÑFÔF‰DˆAˆqØlÐlÐlÐlÐlÐlÐ^kÐlÑlÔlˆMð ð 	ØˆKà1>ˆKÐ-Ñ.ØÐ(Ð-AÐ(Ø<I�Ð8Ñ9ð &ð 8ØŸ)š) A™,œ,�Ø07�Ð,Ñ-àÐõ Œy×%Ò%Ñ'Ô'ð 	DÐ,@ð 	DÀ]ÐE^å ¥ ]°MÑ!BÔ!BÑCÔCˆMàð 	!Ø Ð à�IŠI�a‰LŒLˆà�-ÐÐrU   r0   Ú
prune_normÚ
prune_headc                 ó  — t          t          | j        ¦  «        |¦  «        \  }}| j        d|dz   …         | _        |rt          j        ¦   «         | _        |r.t          j        ¦   «         | _        |                      dd¦  «         |S )aE  Prune layers not required for specified intermediates.

        Args:
            indices: Indices of intermediate layers to keep.
            prune_norm: Whether to prune normalization layer.
            prune_head: Whether to prune the classifier head.

        Returns:
            List of indices that were kept.
        Nr0   r   r<   )r2   rÃ  rp  rn   ro   r  r<  r£  )rx   r³  rÓ  rÔ  rÊ  rË  s         rS   Úprune_intermediate_layersz+VisionTransformer.prune_intermediate_layersž  s~   € õ  #7µs¸4¼;Ñ7GÔ7GÈÑ"QÔ"QÑˆ�iØ”k . 9¨q¡= .Ô1ˆŒØð 	&Ýœ™œˆDŒIØð 	)Ýœ;™=œ=ˆDŒLØ×!Ò! ! RÑ(Ô(Ð(ØÐrU   ÚnrÐ   c           	      óB   — |                       |||||rdndd|¬¦  «        S )aÈ  Get intermediate layer outputs (DINO interface compatibility).

        NOTE: This API is for backwards compat, favour using forward_intermediates() directly.

        Args:
            x: Input tensor.
            n: Number or indices of layers.
            reshape: Reshape to NCHW format.
            return_prefix_tokens: Return prefix tokens.
            norm: Apply normalization.

        Returns:
            List of intermediate features.
        r²  r¹  T)r´  r  rS  r¶  r|   )rÒ  )rx   r{   r×  rÐ   r´  r  r|   s          rS   Úget_intermediate_layersz)VisionTransformer.get_intermediate_layers·  s>   € ð. ×)Ò)ØˆqØ!5ØØ!(Ð3�v�v¨eØ#Øð *ñ 
ô 
ð 	
rU   c                 ó²  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|€|r| j        D ]} ||||¬¦  «        }ŒnP| j        r4t          j                             ¦   «         st          | j        |¦  «        }n|                      |¦  «        }|  
                    |¦  «        }|S )z\Forward pass through feature layers (embeddings, transformer blocks, post-transformer norm).Nr€   )rh  r±  rn  ro  rp  rf  r�   rÄ  rÅ  r6   r  )rx   r{   r|   r}   rÎ  s        rS   Úforward_featuresz"VisionTransformer.forward_features×  sÜ   € ð ×Ò˜QÑÔˆØ�OŠO˜AÑÔˆØ�OŠO˜AÑÔˆØ�MŠM˜!ÑÔˆàÐ  IÐ à”{ð Eð E�Ø�C˜ Y¸)ÐDÑDÔD��ðEàÔ$ð 	­U¬Y×-CÒ-CÑ-EÔ-Eð 	Ý˜tœ{¨AÑ.Ô.ˆAˆAà—’˜A‘”ˆAà�IŠI�a‰LŒLˆØˆrU   r  c                 óÆ   — | j         �1| j        s|dd…| j        d…f         }|                       |¦  «        }|S |€| j        n|}t	          ||| j        | j        ¬¦  «        }|S )z´Apply pooling to feature tokens.

        Args:
            x: Feature tensor.
            pool_type: Pooling type override.

        Returns:
            Pooled features.
        N)r  r  r  )rr  r=  r  r1  r$  )rx   r{   r  s      rS   ÚpoolzVisionTransformer.poolï  sˆ   € ð Œ>Ð%ØÔ+ð 2Ø�a�a�a˜Ô/Ð0Ð0Ð0Ô1�Ø—’˜qÑ!Ô!ˆAØˆHØ(1Ð(9�DÔ$Ð$¸yˆ	ÝØØØ"Ô4Ø"&Ô":ð	
ñ 
ô 
ˆð ˆrU   Ú
pre_logitsc                 ó²   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|r|n|                      |¦  «        S )zÊForward pass through classifier head.

        Args:
            x: Feature tensor.
            pre_logits: Return features before final classifier.

        Returns:
            Output tensor.
        )rÝ  r<  rs  rt  )rx   r{   rÞ  s      rS   Úforward_headzVisionTransformer.forward_head  sL   € ð �IŠI�a‰LŒLˆØ�LŠL˜‰OŒOˆØ�NŠN˜1ÑÔˆØÐ0ˆqˆq D§I¢I¨a¡L¤LÐ0rU   c                 ó`   — |                       |||¬¦  «        }|                      |¦  «        }|S r   )rÛ  rà  r�   s       rS   r‚   zVisionTransformer.forward  s6   € ð ×!Ò! !¨yÀIÐ!ÑNÔNˆØ×Ò˜aÑ Ô ˆØˆrU   rœ   ©r<   T©r<   ©F)Tr–   )NN)	NFFFr²  FFNF)r0   FT)r0   FFFNrƒ   )3r…   r†   r‡   rˆ   r   r‹   rÜ   r   rW   r   r   r   rs   r
   r   rŠ   r   rN   r   r.   r   rn   rŒ   rl   r‚  r”   r�  r�   rÄ  Úignorer”  r	   r—  r   r   r›  rž  r   r£  r©  rŽ   r±  r   rÒ  rÖ  rÙ  rÛ  rÝ  rà  r‚   r�   r�   s   @rS   r;   r;   Ê  sV  ø€ € € € € € ðð ð
 ˜D”kÐ!Ð!Ñ!ð 58Ø68ØØ#ØV]Ø ØØØ!Ø!Ø!Ø$)Ø#(Ø"Ø+/Ø $Ø$Ø#(ØØ"Ø#Ø&*Ø(-Ø%*Ø$)Ø!Ø#%Ø%'Ø$&Ø$&Ø$&ØSUØ"Ø$.Ø48Ø.2Ø-1Ø(-Ø),Ø$-ØØðW1ð 1à˜C  s¨C x¤Ð0Ô1ð1ð ˜c 5¨¨c¨¤?Ð2Ô3ð1ð ð	1ð
 ð1ð !Ð!RÔSð1ð ð1ð ð1ð ð1ð ð1ð ð1ð ð1ð "ð1ð !ð1ð ð1ð  " %œð!1ð" ð#1ð$ ð%1ð& !ð'1ð( ð)1ð* ð+1ð, ð-1ð. ˜d”^ð/1ð0 "&ð11ð2 #ð31ð4 "ð51ð6 ð71ð8 !ð91ð: #ð;1ð< "ð=1ð> "ð?1ð@ "ðA1ðB !Ð!OÔPðC1ðD ðE1ðF "ðG1ðH ' yÔ1ðI1ðJ ! Ô+ðK1ðL   	Ô*ðM1ðN ˜2œ9”oðO1ðP ˜BœI”ðQ1ðR "ðS1ðX 
ðY1ð 1ð 1ð 1ð 1ð 1ðB1ð 1ð 1ð 1ð#ð # ð #¸ð #Èð #ð #ð #ð #ð.!˜rœyð !¨Tð !ð !ð !ð !ð
 „Y×ÒÑÔð
5ð 
5¨sð 
5¸Cð 
5Èð 
5ð 
5ð 
5ñ Ôð
5ð „YÔðE  S¤ð Eð Eð Eñ ÔðEð „YÔð
ð 
 Dð 
°T¸#¸uÀSÈ$ÀYÔ?OÐ:OÔ5Pð 
ð 
ð 
ñ Ôð
ð „YÔð<ð <¨Tð <¸Tð <ð <ð <ñ Ôð<ð „YÔð ¤	ð ð ð ñ Ôðð'ð '¨Cð '¸hÀs¼mð 'ÐW[ð 'ð 'ð 'ð 'ð. 37Ø48ðð à˜u S¨# XœÔ/ðð !  s¨C x¤Ô1ðð 
ð	ð ð ð ð2% ˜EœLð % ¨U¬\ð % ð % ð % ð % ðT 8<Ø).ØØ$Ø$Ø',Ø %Ø04Ø#ð_ ð _ àŒ|ð_ ð ˜e C¨¨c¬ NÔ3Ô4ð_ ð #'ð	_ ð
 ð_ ð ð_ ð ð_ ð !%ð_ ð ð_ ð   ¤Ô-ð_ ð ð_ ð 
ˆt�E”LÔ! 5¨¬°t¸E¼LÔ7IÐ)IÔ#JÈDÐQTÐVYÐQYÌNÐZÔ	[ð_ ð _ ð _ ð _ ðF ./Ø$Ø#ð	ð à˜3  S¤	˜>Ô*ðð ðð ð	ð
 
ˆcŒðð ð ð ð8 45Ø!Ø).ØØ04ð
ð 
àŒ|ð
ð �S˜$˜sœ) U¨3¤ZÐ/Ô0ð
ð ð	
ð
 #'ð
ð ð
ð   ¤Ô-ð
ð 
ˆeŒlÔ	ð
ð 
ð 
ð 
ðF 15Ø#ð	ð àŒ|ðð   ¤Ô-ðð ð	ð
 
Œðð ð ð ð0ð �e”lð ¨x¸¬}ð ÈÌð ð ð ð ð01ð 1˜eœlð 1¸ð 1ÈÌð 1ð 1ð 1ð 1ð$ 15Ø#ð	ð àŒ|ðð   ¤Ô-ðð ð	ð
 
Œðð ð ð ð ð ð ð rU   r<   r[  Únamera  c                 ód  — t          | t          j        ¦  «        rEt          | j        d¬¦  «         | j        �&t          j                             | j        ¦  «         dS dS t          | d¦  «        r|  	                    ¦   «          dS |r&t          | d¦  «        r|  
                    ¦   «          dS dS dS )zðViT weight initialization, original timm impl (for reproducibility).

    Args:
        module: Module to initialize.
        name: Module name for context.
        needs_reset: If True, call reset_parameters() on modules that have it.
    r†  r‡  Nr”   r¹   )rM   rn   rª   r$   r™   rh   r—   r¾   rj  r”   r¹   ©r[  ræ  ra  s      rS   rŽ  rŽ  !  sÉ   € õ �&�"œ)Ñ$Ô$ð "Ý�f”m¨Ð-Ñ-Ô-Ð-ØŒ;Ð"ÝŒG�NŠN˜6œ;Ñ'Ô'Ð'Ð'Ð'ð #Ð"å	�˜Ñ	(Ô	(ð "Ø×ÒÑÔÐÐÐØ	ð "� Ð);Ñ<Ô<ð "Ø×ÒÑ!Ô!Ð!Ð!Ð!ð"ð "ð "ð "rU   r‹  c                 óN  — t          | t          j        ¦  «        rÝ|                     d¦  «        rKt          j                             | j        ¦  «         t          j                             | j        |¦  «         dS t          j         	                    | j        ¦  «         | j        �Pd|v r&t          j         
                    | j        d¬¦  «        n#t          j                             | j        ¦  «         dS dS t          | t          j        ¦  «        rCt          | j        ¦  «         | j        �&t          j                             | j        ¦  «         dS dS t          | d¦  «        r|                      ¦   «          dS |r&t          | d¦  «        r|                      ¦   «          dS dS dS )a  ViT weight initialization, matching JAX (Flax) impl.

    Args:
        module: Module to initialize.
        name: Module name for context.
        head_bias: Bias value for head layer.
        needs_reset: If True, call reset_parameters() on modules that have it.
    rt  Nrt   rˆ  r‡  r”   r¹   )rM   rn   rª   Ú
startswithr—   r¾   r™   r˜   rh   Úxavier_uniform_r÷   ÚConv2dr%   rj  r”   r¹   )r[  ræ  r‹  ra  s       rS   Úinit_weights_vit_jaxrí  3  s†  € õ �&�"œ)Ñ$Ô$ð "Ø�?Š?˜6Ñ"Ô"ð 	iÝŒG�NŠN˜6œ=Ñ)Ô)Ð)ÝŒG×Ò˜fœk¨9Ñ5Ô5Ð5Ð5Ð5åŒG×#Ò# F¤MÑ2Ô2Ð2ØŒ{Ð&Ø:?À4¸-¸-•”—’ ¤°�Ñ6Ô6Ð6ÍRÌWÏ^Ê^Ð\bÔ\gÑMhÔMhøè è ð 'Ð&å	�F�BœIÑ	&Ô	&ð "Ý�f”mÑ$Ô$Ð$ØŒ;Ð"ÝŒG�NŠN˜6œ;Ñ'Ô'Ð'Ð'Ð'ð #Ð"å	�˜Ñ	(Ô	(ð "Ø×ÒÑÔÐÐÐØ	ð "� Ð);Ñ<Ô<ð "Ø×ÒÑ!Ô!Ð!Ð!Ð!ð"ð "ð "ð "rU   c                 ól  — t          | t          j        ¦  «        rÉd|v rrt          j        dt          | j        j        d         dz  | j        j        d         z   ¦  «        z  ¦  «        }t          j         	                    | j        | |¦  «         n$t          j         
                    | j        ¦  «         | j        �&t          j                             | j        ¦  «         dS dS t          | d¦  «        r|                      ¦   «          dS |r&t          | d¦  «        r|                      ¦   «          dS dS dS )	zôViT weight initialization, matching moco-v3 impl minus fixed PatchEmbed.

    Args:
        module: Module to initialize.
        name: Module name for context.
        needs_reset: If True, call reset_parameters() on modules that have it.
    Úqkvg      @r   r¥   r0   Nr”   r¹   )rM   rn   rª   rð   r|  rŠ   r™   rÇ   r—   Úuniform_rë  rh   r¾   rj  r”   r¹   )r[  ræ  ra  Úvals       rS   Úinit_weights_vit_mocorò  S  s1  € õ �&�"œ)Ñ$Ô$ð "Ø�Dˆ=ˆ=å”)˜B¥ v¤}Ô':¸1Ô'=ÀÑ'BÀVÄ]ÔEXÐYZÔE[Ñ'[Ñ!\Ô!\Ñ\Ñ]Ô]ˆCÝŒG×Ò˜Vœ]¨S¨D°#Ñ6Ô6Ð6Ð6åŒG×#Ò# F¤MÑ2Ô2Ð2ØŒ;Ð"ÝŒG�NŠN˜6œ;Ñ'Ô'Ð'Ð'Ð'ð #Ð"å	�˜Ñ	(Ô	(ð "Ø×ÒÑÔÐÐÐØ	ð "� Ð);Ñ<Ô<ð "Ø×ÒÑ!Ô!Ð!Ð!Ð!ð"ð "ð "ð "rU   c                 óZ   — |r&t          | d¦  «        r|                      ¦   «          d S d S d S )Nr¹   )rj  r¹   rè  s      rS   Úinit_weights_reset_parametersrô  j  sH   € Øð "•w˜vÐ'9Ñ:Ô:ð "Ø×ÒÑ!Ô!Ð!Ð!Ð!ð"ð "ð "ð "rU   rH  rƒ  c                 ó  — |                       d¦  «        rt          t          ||¬¦  «        S |                       d¦  «        rt          t          |¬¦  «        S | dk    rt          t          |¬¦  «        S t          t
          |¬¦  «        S )NrH  )r‹  ra  rJ  r`  rG  )rê  r   rí  rò  rô  rŽ  )rƒ  r‹  ra  s      rS   rŠ  rŠ  o  s‰   € Ø‡‚�uÑÔð 	GÝÕ+°yÈkÐZÑZÔZÐZØ	�Š˜Ñ	 Ô	 ð GÝÕ,¸+ÐFÑFÔFÐFØ	�ŠˆåÕ4À+ÐNÑNÔNÐNõ Õ,¸+ÐFÑFÔFÐFrU   rj   ÚbicubicÚposembÚ
posemb_newÚgs_newÚinterpolationÚ	antialiasc           	      ó   — |j         d         |z
  }| j         d         |z
  }t          t          j        |¦  «        ¦  «        gdz  }t	          |¦  «        s%t          t          j        |¦  «        ¦  «        gdz  }t          | |||||d¬¦  «        S )z¨ Rescale the grid of position embeddings when loading from state_dict.
    *DEPRECATED* This function is being deprecated in favour of using resample_abs_pos_embed
    r0   rÂ   T)r  rú  rû  r§  )rÇ   rs   rð   r|  rÃ  r'   )	r÷  rø  r  rù  rú  rû  Úntok_newÚntok_oldÚgs_olds	            rS   Úresize_pos_embedr   |  sž   € ð Ô Ô"Ð%6Ñ6€HØŒ|˜AŒÐ!2Ñ2€HÝ•$”)˜HÑ%Ô%Ñ&Ô&Ð'¨!Ñ+€FÝˆv‰;Œ;ð 0Ý•d”i Ñ)Ô)Ñ*Ô*Ð+¨aÑ/ˆÝ!Ø�˜Ø+Ø#ØØðñ ô ð rU   Úmodelr�  r‘  Úload_bfloat16c                 ó #  ‡‡&‡'‡(‡)‡*‡+‡,— ddl Š+‰r
ddlm Š( ddlŠ*d\ˆ(ˆˆ*ˆ+fd„	Š&|r|› d�f}nd}t          ||‰r‰(j        n‰+j        t          | ¦  «        j        › d�¬¦  «        Š,d	}d
}|sd‰,v rd}nd‰,v rd}nd‰,v rd}|                     d¦  «        p|› d�‰,v p|› d�‰,v }t          | d¦  «        ot          | j	        d¦  «        }|�r4| j	        }	 ‰&‰,|› d�         ¦  «        }
d}|
j
        dk    r|
}nÏ|
j
        dk    rªt                               d|	j        f¦  «        D ]‡}t          |
j        d         |¦  «        \  }}t!          t#          j        |¦  «        ¦  «        }|sD||z  |k    r;|
                     |
j        d         |||¦  «                             dddd¦  «        } nŒˆnt+          d|› d|
j
        › �¦  «        ‚|�˜t-          |	j        |¦  «        }|j        dd…         |	j        k    rt1          ||	j        ||d¬¦  «        }|	j        rI|	j        r,|                     dddd¦  «                             d¦  «        }
n|                     d¦  «        }
n|}
|
j        |	j        j        j        k    rFt+          d|› dt=          |
j        ¦  «        › dt=          |	j        j        j        ¦  «        › �¦  «        ‚|	j        j                             |
¦  «         |	j        j         �8|› d �‰,v r1|	j        j                               ‰&‰,|› d �         ¦  «        ¦  «         |	j!        �5|› d!�‰,v r.|	j!                              ‰&‰,|› d!�         d
¬"¦  «        ¦  «         |r|› d�n|› d#�}|	j"        ��‹|‰,v �r† ‰&‰,|         d
¬"¦  «        }d}|j
        dk    r| #                    d¦  «        }|j
        dk    �r|j        d         dk    rà|j        d         }t!          t#          j        |¦  «        ¦  «        }||z  |k    r…|› d!�‰,v rdntI          |	d$d¦  «        }||z  }t!          t#          j        |¦  «        ¦  «        }||z  |k    r t+          d%|j        d         › d&|› �¦  «        ‚|dd…d|…f         }|dd…|d…f         }|                     d|||j        d'         ¦  «        }n| #                    d¦  «        }|j
        dk    rt+          d(|› d|j        › �¦  «        ‚|�Ãd}|	j!        �?|j        d         |k    r.|	j!         %                    |dd…||dz   …f         ¦  «         |dz  }|	j&        �t|j        d         |k    rctO          |	j&        j        d         |j        d         |z
  ¦  «        }|	j&        dd…d|…f          %                    |dd…|||z   …f         ¦  «         |j        |	j"        j        k    r&tQ          ||	j"        j        dd…         ||d¬)¦  «        }|	j"                             |¦  «         �n­t          | j)        d*¦  «        �rÌ| j)        j*        }t          |d+¦  «         }|r|n|j+        }|j,        j                             t-          |j,        j        j        d          ‰&‰,|› d,�         ¦  «        ¦  «        ¦  «         |j-        j                              ‰&‰,|› d-�         ¦  «        ¦  «         |j-        j                               ‰&‰,|› d.�         ¦  «        ¦  «         |�sÖt]          |j/        ¦  «        D �]À\  }}t]          |j0        ¦  «        D �]¤\  }}|› d/|dz   › d0|dz   › d1�}tc          d¦  «        D ]Ô} tI          |d2| dz   › �¦  «        j                              ‰&‰,|› d2| dz   › d3�         ¦  «        ¦  «         tI          |d4| dz   › �¦  «        j                              ‰&‰,|› d5| dz   › d6�         ¦  «        ¦  «         tI          |d4| dz   › �¦  «        j                               ‰&‰,|› d5| dz   › d7�         ¦  «        ¦  «         ŒÕ|j2        �¢|j2        j,        j                              ‰&‰,|› d8�         ¦  «        ¦  «         |j2        j-        j                              ‰&‰,|› d9�         ¦  «        ¦  «         |j2        j-        j                               ‰&‰,|› d:�         ¦  «        ¦  «         �Œ¦�ŒÂ ‰&‰,|› d�         ¦  «        }n<t-          | j)        j        j        j        d          ‰&‰,|› d�         ¦  «        ¦  «        }|j        dd…         | j)        j        j        j        dd…         k    r0t1          || j)        j        j        j        dd…         ||d¬¦  «        }| j)        j        j                             |¦  «         | j)        j        j                               ‰&‰,|› d �         ¦  «        ¦  «         | j!        �.| j!                              ‰&‰,|› d!�         d
¬"¦  «        ¦  «         |r ‰&‰,|› d�         d
¬"¦  «        }n ‰&‰,|› d#�         d
¬"¦  «        }|j        | j"        j        k    rCtI          | d;d
¦  «        rdntI          | d$d¦  «        }!tg          || j)        j4        |!||d¬<¦  «        }| j"                             |¦  «         t          | j-        d=¦  «        rb| j-        j                              ‰&‰,|› d>�         ¦  «        ¦  «         | j-        j                               ‰&‰,|› d?�         ¦  «        ¦  «         tk          | j6        tn          j8        ¦  «        r˜|› d@�‰,v r‘| j6        j         j        d         ‰,|› d@�         j        d'         k    rb| j6        j                              ‰&‰,|› dA�         ¦  «        ¦  «         | j6        j                               ‰&‰,|› d@�         ¦  «        ¦  «         tk          | j9        tt          ¦  «        �r÷|› dB�}"|"dCz   Š)| j9        j;                              ‰&‰,|"› dD�         d
¬"¦  «        ¦  «         | j9        j<        j                             t{          j>        ˆ&ˆ)ˆ,fdE„dFD ¦   «         ¦  «        ¦  «         | j9        j<        j                              t{          j>        ˆ&ˆ)ˆ,fdG„dFD ¦   «         ¦  «        ¦  «         | j9        j?        j                              ‰&‰,‰)› dH�         d
¬"¦  «                             d¦  «        j@        ¦  «         | j9        j?        j                               ‰&‰,‰)› dI�         d
¬"¦  «                             d'¦  «        ¦  «         | j9        j        j                              ‰&‰,‰)› dJ�         ¦  «                             d¦  «        ¦  «         | j9        j        j                               ‰&‰,‰)› dK�         ¦  «        ¦  «         | j9        j-        j                              ‰&‰,|"› dL�         ¦  «        ¦  «         | j9        j-        j                               ‰&‰,|"› dM�         ¦  «        ¦  «         tc          d¦  «        D ]œ} tI          | j9        jA        dN| dz   › �¦  «        j                              ‰&‰,|"› dO| › d3�         ¦  «        ¦  «         tI          | j9        jA        dN| dz   › �¦  «        j                               ‰&‰,|"› dO| › d7�         ¦  «        ¦  «         Œ�|rdPndQ\  }#}$}%t]          | j0         B                    ¦   «         ¦  «        D �]³\  }}|› d�‰,v r|› dR�}"|Š'n
|› dS|› d1�}"dŠ'|"dT|#› d1�z   Š)|jC        j                              ‰&‰,|"› dL�         ‰'¬U¦  «        ¦  «         |jC        j                               ‰&‰,|"› dM�         ‰'¬U¦  «        ¦  «         |jD        jE        j                             t{          j>        ˆ&ˆ'ˆ)ˆ,fdV„dWD ¦   «         ¦  «        ¦  «         |jD        jE        j                              t{          j>        ˆ&ˆ'ˆ)ˆ,fdX„dWD ¦   «         ¦  «        ¦  «         |jD        j        j                              ‰&‰,‰)› dJ�         ‰'¬U¦  «                             d¦  «        ¦  «         |jD        j        j                               ‰&‰,‰)› dK�         ‰'¬U¦  «        ¦  «         |jF        j                              ‰&‰,|"› dY|%› d6�         ‰'¬U¦  «        ¦  «         |jF        j                               ‰&‰,|"› dY|%› d7�         ‰'¬U¦  «        ¦  «         tc          d¦  «        D ]œ} tI          |jA        dN| dz   › �¦  «        j                              ‰&‰,|"› dZ|$› d[| › d3�         ‰'¬U¦  «        ¦  «         tI          |jA        dN| dz   › �¦  «        j                               ‰&‰,|"› dZ|$› d[| › d7�         ‰'¬U¦  «        ¦  «         Œ��ŒµdS )]z@Load weights from official Google JAX/Flax ``.npz`` checkpoints.r   NTc                 óX  •— |�| |         } ‰rG|                       ‰j        ¦  «                             ‰j        ¦  «        } ‰                     | ¦  «        } | j        dk    rL| j        d         | j        d         cxk    r| j        d         cxk    rdk    rn n|                      ¦   «         } |rh| j        dk    r|                      g d¢¦  «        } nE| j        dk    r|                      g d¢¦  «        } n"| j        dk    r|                      ddg¦  «        } t          j
        | ¦  «        } | S )Né   r   r0   rÂ   )r¥   rÂ   r   r0   r¥   )rÂ   r   r0   )rÉ   Úbfloat16Úastyperæ   ÚarrayÚndimrÇ   ÚflattenrÊ   r�   Ú
from_numpy)Ú_wÚtÚidxÚjnpr  Ú	ml_dtypesÚnps      €€€€rS   Ú_n2pz_load_weights.<locals>._n2p�  s&  ø€ Øˆ?Ø�C”ˆBàð 	Ø—’˜Ô+Ñ,Ô,×3Ò3°C´KÑ@Ô@ˆBØ—’˜"‘”ˆBàŒ7�aŠ<ˆ<˜BœH QœK¨2¬8°A¬;ÐJÐJÒJÐJ¸"¼(À1¼+ÐJÐJÒJÐJÈÒJÐJÐJÐJÐJØ—’‘”ˆBØð 	*ØŒw˜!Š|ˆ|Ø—\’\ , , ,Ñ/Ô/��Ø”˜A’�Ø—\’\ ) ) )Ñ,Ô,��Ø”˜A’�Ø—\’\ 1 a &Ñ)Ô)�åÔ˜bÑ!Ô!ˆØˆ	rU   úembedding/kernel)r  úopt/target/embedding/kernelúparams/embedding/kernelúparams/img/embedding/kernelz.load_pretrained())Úrequired_anyÚload_fnÚcontextÚbilinearFr  zopt/target/r  zparams/r  zparams/img/Úpos_embeddingz*Transformer/encoderblock/LayerNorm_0/scaleÚembedsr}  r  rÂ   r¥   r0   z$Unsupported patch embedding rank in z: rÅ   ©rú  rû  r§  z"Patch embedding shape mismatch in z: checkpoint=z, model=zembedding/biasÚcls©r  z(Transformer/posembed_input/pos_embeddingr  z Cannot infer position grid from z tokens in rÀ   z(Unsupported position embedding shape in )r¥  rú  rû  r§  Úbackbonerš  zconv_root/kernelzgn_root/scalezgn_root/biasÚblockz/unitú/Úconvú/kernelr  Úgnz/scaleú/biaszconv_proj/kernelzgn_proj/scalezgn_proj/biasr8  ©r¥  r  rú  rû  r§  r™   zTransformer/encoder_norm/scalezTransformer/encoder_norm/biasz	head/biaszhead/kernelz
MAPHead_0/zMultiHeadDotProductAttention_0/Úprobec                 óp   •— g | ]2} ‰‰‰› |› d �         d¬¦  «                              d¦  «        j        ‘Œ3S )r$  Fr  r0   ©r
  ÚT©r  r×  r  Ú
mha_prefixÚws     €€€rS   r  z!_load_weights.<locals>.<listcomp>}  s_   ø€ ð 3`ð 3`ð 3`ØIJˆDˆD��jÐ, !Ð,Ð,Ð,Ô-°Ð7Ñ7Ô7×?Ò?ÀÑBÔBÔDð3`ð 3`ð 3`rU   )ÚkeyÚvaluec                 óf   •— g | ]-} ‰‰‰› |› d �         d¬¦  «                              d¦  «        ‘Œ.S )r&  Fr  rÀ   ©rÐ   r,  s     €€€rS   r  z!_load_weights.<locals>.<listcomp>  s\   ø€ ð 1]ð 1]ð 1]ØFGˆDˆD��jÐ* !Ð*Ð*Ð*Ô+¨uÐ5Ñ5Ô5×=Ò=¸bÑAÔAð1]ð 1]ð 1]rU   zquery/kernelz
query/biasz
out/kernelzout/biaszLayerNorm_0/scalezLayerNorm_0/biasÚfczMlpBlock_0/Dense_)r   r   r0   )r0   r¥   rÂ   zTransformer/encoderblock/zTransformer/encoderblock_ÚMultiHeadDotProductAttention_)r  c                 ór   •— g | ]3} ‰‰‰› |› d �         d‰¬¦  «                              d¦  «        j        ‘Œ4S )r$  F©r  r  r0   r*  ©r  r×  r  r  r-  r.  s     €€€€rS   r  z!_load_weights.<locals>.<listcomp>–  sa   ø€ ð /rð /rð /rØRSˆDˆD��jÐ, !Ð,Ð,Ð,Ô-°¸CÐ@Ñ@Ô@×HÒHÈÑKÔKÔMð/rð /rð /rrU   )Úqueryr/  r0  c                 óh   •— g | ].} ‰‰‰› |› d �         d‰¬¦  «                              d¦  «        ‘Œ/S )r&  Fr6  rÀ   r2  r7  s     €€€€rS   r  z!_load_weights.<locals>.<listcomp>˜  s^   ø€ ð -oð -oð -oØOPˆDˆD��jÐ* !Ð*Ð*Ð*Ô+¨u¸#Ð>Ñ>Ô>×FÒFÀrÑJÔJð-oð -oð -orU   Ú
LayerNorm_Ú	MlpBlock_z/Dense_)TN)GÚnumpyÚ	jax.numpyr  r3   ÚloadÚtyper…   rê  rj  r  r	  r_  Úfromkeysr/  ÚdivmodrÇ   rs   rð   r|  rÐ   r¾  Ú
ValueErrorr7   r.  r&   Ú	is_linearÚchannels_lastr
  r}  r™   ÚtupleÚcopy_rh   rk  r7  Ú	unsqueezeÚgetattrÚadd_rl  Úminr(   rh  r   rš  r#  r  r{  Ústagesrp  r  Ú
downsampler'   r¨  rM   rt  rn   rª   rr  r   ÚlatentÚkvr�   rÑ   r×   r+  rt   Úchildrenrm   r=   rï  rr   )-r  r�  r‘  r  r  rú  rû  Ú
big_visionÚ	is_naflexr  Úembed_wÚembed_conv_wÚcheckpoint_in_chansÚ
patch_areaÚ	remainderr.  Úpos_embed_keyÚpos_embed_wÚprefix_pos_embedÚnum_pos_tokensr¨  Úcheckpoint_prefix_tokensÚprefix_indexrd  r   Ú	stem_onlyrš  rW  ÚstageÚjr!  ÚbpÚrr  Úblock_prefixÚmha_subÚb_subÚln1_subr  r  r  r-  r  r  r.  s-      `                                  @@@@@@@rS   r“  r“  •  s­  øøøøøøøø€ ð ÐÐÐØð ØÐÐÐÐÐØÐÐÐðð ð ð ð ð ð ð ð ð* ð 
Ø!Ð3Ð3Ð3Ð5ˆˆð
ˆõ 	ØØ!Ø)Ð6�”�¨r¬wÝ˜‘;”;Ô'Ð;Ð;Ð;ð		ñ 	ô 	€Að €MØ€IØð #Ø(¨AÐ-Ð-Ø"ˆFˆFØ&¨!Ð+Ð+ØˆFˆFØ*¨aÐ/Ð/Ø"ˆFà×Ò˜)Ñ$Ô$ð 	CØÐ Ð Ð  AÐ%ð	CàÐ=Ð=Ð=ÀÐBð õ
 ˜˜xÑ(Ô(ÐJ­W°U´\À6Ñ-JÔ-J€Iàñ Z+ð ”ˆØ�$�q˜FÐ4Ð4Ð4Ô5Ñ6Ô6ˆØˆØŒ<˜1ÒÐØ"ˆLˆLØŒ\˜QÒÐõ (,§}¢}°a¸¼Ð5IÑ'JÔ'Jð ð Ð#Ý(.¨w¬}¸QÔ/?ÐATÑ(UÔ(UÑ%�
˜IÝ ¥¤¨:Ñ!6Ô!6Ñ7Ô7�
Ø ð  Z°*Ñ%<À
Ò%JÐ%JØ#*§?¢?Øœ aÔ(¨*°jÐBUñ$Wô $WßW^ÒW^Ð_`ÐbcÐefÐhiÑWjÔWjð !à�EøøåÐeÀOÐeÐeÐW^ÔWcÐeÐeÑfÔfÐfàÐ#Ý+¨F¬O¸\ÑJÔJˆLØÔ! " # #Ô&¨&Ô*;Ò;Ð;Ý3Ø ØÔ%Ø"/Ø'Ø ð ñ  ô  �ð Ôð 'ØÔ'ð 6Ø*×2Ò2°1°a¸¸AÑ>Ô>×FÒFÀqÑIÔI�G�Gà*×2Ò2°1Ñ5Ô5�G�Gà&�àŒ=˜FœKÔ.Ô4Ò4Ð4Ýð^°_ð ^ð ^Ý# G¤MÑ2Ô2ð^ð ^Ý<AÀ&Ä+ÔBTÔBZÑ<[Ô<[ð^ð ^ñ_ô _ð _ð 	ŒÔ× Ò  Ñ)Ô)Ð)ØŒ;ÔÐ'¨vÐ,EÐ,EÐ,EÈÐ,JÐ,JØŒKÔ×"Ò" 4 4¨¨VÐ*CÐ*CÐ*CÔ(DÑ#EÔ#EÑFÔFÐFàÔÐ'¨v¨N¨N¨N¸aÐ,?Ð,?ØÔ×"Ò" 4 4¨¨V¨.¨.¨.Ô(9¸UÐ#CÑ#CÔ#CÑDÔDÐDð )3ð EˆvÐ$Ð$Ð$Ð$ØÐDÐDÐDð 	ð ÔÑ'¨M¸QÐ,>Ñ,>Ø˜$˜q Ô/°5Ð9Ñ9Ô9ˆKØ#ÐØÔ 1Ò$Ð$Ø)×3Ò3°AÑ6Ô6�ØÔ 1Ò$Ñ$ØÔ$ QÔ'¨1Ò,Ð,à%0Ô%6°qÔ%9�NÝ #¥D¤I¨nÑ$=Ô$=Ñ >Ô >�IØ  9Ñ,°Ò>Ð>à$*   °1Ð!4Ð!4˜A˜A½'À&ÐJ]Ð_`Ñ:aÔ:að 1à&Ð*BÑB˜Ý$'­¬	°.Ñ(AÔ(AÑ$BÔ$B˜	Ø$ yÑ0°NÒBÐBÝ",ð!8À;ÔCTÐUVÔCWð !8ð !8Ø&5ð!8ð !8ñ#9ô #9ð 9ð ,7°q°q°qÐ:SÐ;SÐ:SÐ7SÔ+TÐ(Ø&1°!°!°!Ð5MÐ5NÐ5NÐ2NÔ&O˜Ø"-×"5Ò"5°a¸ÀIÈ{ÔO`ÐacÔOdÑ"eÔ"e�K�Kð #.×"7Ò"7¸Ñ":Ô":�KØÔ 1Ò$Ð$Ý Ð!rÈOÐ!rÐ!rÐ_jÔ_pÐ!rÐ!rÑsÔsÐsàÐ+Ø �ØÔ#Ð/Ð4DÔ4JÈ1Ô4MÐP\Ò4\Ð4\ØÔ$×)Ò)Ð*:¸1¸1¸1¸lÈ<ÐZ[ÑK[Ð>[Ð;[Ô*\Ñ]Ô]Ð]Ø  AÑ%�LØÔ#Ð/Ð4DÔ4JÈ1Ô4MÐP\Ò4\Ð4\Ý%(¨Ô)9Ô)?ÀÔ)BÐDTÔDZÐ[\ÔD]Ð`lÑDlÑ%mÔ%m�NØÔ$ Q Q Q¨¨¨Ð%7Ô8×=Ò=Ø(¨¨¨¨L¸ÈÑ9VÐ,VÐ)VÔWñYô Yð Yð Ô  FÔ$4Ô$:Ò:Ð:Ý9ØØ#Ô-Ô3°A°a°CÔ8Ø"/Ø'Ø ðñ ô �ð Ô×"Ò" ;Ñ/Ô/Ð/ùå�5Ô$ jÑ1Ô1ñ 	^àÔ(Ô1ˆHÝ# H¨fÑ5Ô5Ð5ˆIØ(Ð;�8�8¨h¬mˆDØŒIÔ×"Ò"Ý  ¤Ô!1Ô!7¸Ô!:¸D¸DÀÀfÐC^ÐC^ÐC^ÔA_Ñ<`Ô<`ÑaÔañcô cð càŒIÔ×"Ò" 4 4¨¨VÐ*BÐ*BÐ*BÔ(CÑ#DÔ#DÑEÔEÐEØŒIŒN× Ò    a¨6Ð(?Ð(?Ð(?Ô&@Ñ!AÔ!AÑBÔBÐBØñ [Ý )¨(¬/Ñ :Ô :ð 
[ñ 
[‘H�A�uÝ$-¨e¬lÑ$;Ô$;ð 	[ñ 	[™˜˜5Ø &ÐAÐA¨Q°©UÐAÐA¸¸Q¹ÐAÐAÐA˜Ý!& q¡¤ð fð f˜AÝ# E¨>°!°a±%¨>¨>Ñ:Ô:ÔA×GÒGÈÈÈQÐRTÐOhÐOhÐZ[Ð^_ÑZ_ÐOhÐOhÐOhÔMiÑHjÔHjÑkÔkÐkÝ# E¨>°!°a±%¨>¨>Ñ:Ô:ÔA×GÒGÈÈÈQÐRTÐOeÐOeÐXYÐ\]ÑX]ÐOeÐOeÐOeÔMfÑHgÔHgÑhÔhÐhÝ# E¨>°!°a±%¨>¨>Ñ:Ô:Ô?×EÒEÀdÀdÈ1ÐPRÐMbÐMbÐVWÐZ[ÑV[ÐMbÐMbÐMbÔKcÑFdÔFdÑeÔeÐeÐeØ Ô+Ð7Ø!Ô,Ô1Ô8×>Ò>¸t¸tÀAÈÐF]ÐF]ÐF]ÔD^Ñ?_Ô?_Ñ`Ô`Ð`Ø!Ô,Ô1Ô8×>Ò>¸t¸tÀAÈÐFZÐFZÐFZÔD[Ñ?\Ô?\Ñ]Ô]Ð]Ø!Ô,Ô1Ô6×<Ò<¸T¸TÀ!ÀrÐDWÐDWÐDWÔBXÑ=YÔ=YÑZÔZÐZùñ	[ð  ˜4  VÐ"=Ð"=Ð"=Ô >Ñ?Ô?ˆLˆLå+ØÔ!Ô&Ô-Ô3°AÔ6¸¸¸QÀ&Ð?ZÐ?ZÐ?ZÔ=[Ñ8\Ô8\ñ^ô ^ˆLð Ô˜b˜c˜cÔ" eÔ&7Ô&<Ô&CÔ&IÈ"È#È#Ô&NÒNÐNÝ/ØØÔ!Ô&Ô-Ô3°B°C°CÔ8Ø+Ø#Øðñ ô ˆLð 	ÔÔÔ%×+Ò+¨LÑ9Ô9Ð9ØÔÔÔ#×)Ò)¨$¨$¨q°FÐ1JÐ1JÐ1JÔ/KÑ*LÔ*LÑMÔMÐMØŒ?Ð&ØŒO×!Ò! $ $ q¨F¨¨¨Ô'8¸EÐ"BÑ"BÔ"BÑCÔCÐCØð 	`Ø˜$˜q FÐ!9Ð!9Ð!9Ô:¸eÐDÑDÔDˆKˆKà˜$˜q FÐ!TÐ!TÐ!TÔUÐY^Ð_Ñ_Ô_ˆKØÔ ¤Ô 5Ò5Ð5Ý%,¨UÐ4DÀeÑ%LÔ%LÐ x  ÕRYÐZ_ÐatÐvwÑRxÔRxÐÝ0ØØÔ*Ô4Ø"3Ø+Ø#Øðñ ô ˆKð 	Œ×Ò˜kÑ*Ô*Ð*ÝˆuŒz˜8Ñ$Ô$ð QØŒ
Ô×Ò   Q¨&Ð'PÐ'PÐ'PÔ%QÑ RÔ RÑSÔSÐSØŒ
Œ×Ò˜d˜d 1¨Ð%MÐ%MÐ%MÔ#NÑOÔOÑPÔPÐPÝ�5”:�rœyÑ)Ô)ð =ØÐ Ð Ð  AÐ%Ð%ØŒJŒOÔ! !Ô$¨¨VÐ*>Ð*>Ð*>Ô(?Ô(EÀbÔ(IÒIÐIØŒ
Ô×Ò   Q¨&Ð'=Ð'=Ð'=Ô%>Ñ ?Ô ?Ñ@Ô@Ð@ØŒ
Œ×Ò˜d˜d 1¨Ð%9Ð%9Ð%9Ô#:Ñ;Ô;Ñ<Ô<Ð<õ
 �%”/Õ#6Ñ7Ô7ñ wØ Ð,Ð,Ð,ˆØ!Ð$FÑFˆ
ØŒÔ×$Ò$ T T¨!¨|Ð,BÐ,BÐ,BÔ*CÀuÐ%MÑ%MÔ%MÑNÔNÐNØŒÔÔ!×'Ò'­¬	ð 3`ð 3`ð 3`ð 3`ð 3`ð 3`ØN^ð3`ñ 3`ô 3`ñ )aô )añ 	bô 	bð 	bàŒÔÔ×%Ò%¥e¤ið 1]ð 1]ð 1]ð 1]ð 1]ð 1]ØK[ð1]ñ 1]ô 1]ñ '^ô '^ñ 	_ô 	_ð 	_àŒÔÔ ×&Ò& t t¨A°Ð.IÐ.IÐ.IÔ,JÈeÐ'TÑ'TÔ'T×'\Ò'\Ð]^Ñ'_Ô'_Ô'aÑbÔbÐbØŒÔÔ×$Ò$ T T¨!¨zÐ,EÐ,EÐ,EÔ*FÈ%Ð%PÑ%PÔ%P×%XÒ%XÐY[Ñ%\Ô%\Ñ]Ô]Ð]ØŒÔÔ#×)Ò)¨$¨$¨q°JÐ1JÐ1JÐ1JÔ/KÑ*LÔ*L×*TÒ*TÐUVÑ*WÔ*WÑXÔXÐXØŒÔÔ!×'Ò'¨¨¨Q°*Ð/FÐ/FÐ/FÔ-GÑ(HÔ(HÑIÔIÐIØŒÔÔ#×)Ò)¨$¨$¨q°LÐ1SÐ1SÐ1SÔ/TÑ*UÔ*UÑVÔVÐVØŒÔÔ!×'Ò'¨¨¨Q°,Ð/PÐ/PÐ/PÔ-QÑ(RÔ(RÑSÔSÐSÝ�q‘”ð 	wð 	wˆAÝ�E”OÔ'¨¨a°!©e¨¨Ñ6Ô6Ô=×CÒCÀDÀDÈÈlÐKwÐKwÐmnÐKwÐKwÐKwÔIxÑDyÔDyÑzÔzÐzÝ�E”OÔ'¨¨a°!©e¨¨Ñ6Ô6Ô;×AÒAÀ$À$ÀqÈLÐIsÐIsÐklÐIsÐIsÐIsÔGtÑBuÔBuÑvÔvÐvÐvà+5ÐD˜i˜i¸9Ñ€GˆU�GÝ˜eœl×3Ò3Ñ5Ô5Ñ6Ô6ð Tñ T‰ˆˆ5ØÐ@Ð@Ð@ÀAÐEÐEØ$Ð?Ð?Ð?ˆLØˆCˆCà$ÐCÐC¸qÐCÐCÐCˆLØˆCØ!Ð$NÀGÐ$NÐ$NÐ$NÑNˆ
ØŒÔ× Ò    a¨<Ð(JÐ(JÐ(JÔ&KÐQTÐ!UÑ!UÔ!UÑVÔVÐVØŒÔ×Ò˜t˜t A¨Ð&GÐ&GÐ&GÔ$HÈcÐRÑRÔRÑSÔSÐSØŒ
ŒÔ×#Ò#¥E¤Ið /rð /rð /rð /rð /rð /rð /rØWpð/rñ /rô /rñ %sô %sñ 	tô 	tð 	tàŒ
ŒÔ×!Ò!¥%¤)ð -oð -oð -oð -oð -oð -oð -oØTmð-oñ -oô -oñ #pô #pñ 	qô 	qð 	qàŒ
ŒÔ×$Ò$ T T¨!¨zÐ,EÐ,EÐ,EÔ*FÈCÐ%PÑ%PÔ%P×%XÒ%XÐYZÑ%[Ô%[Ñ\Ô\Ð\ØŒ
ŒÔ×"Ò" 4 4¨¨ZÐ*AÐ*AÐ*AÔ(BÈÐ#LÑ#LÔ#LÑMÔMÐMØŒÔ× Ò    a¨<Ð(RÐ(RÀ7Ð(RÐ(RÐ(RÔ&SÐY\Ð!]Ñ!]Ô!]Ñ^Ô^Ð^ØŒÔ×Ò˜t˜t A¨Ð&OÐ&OÀÐ&OÐ&OÐ&OÔ$PÐVYÐZÑZÔZÑ[Ô[Ð[Ý�q‘”ð 	Tð 	TˆAÝ�E”I˜| A¨¡E˜|˜|Ñ,Ô,Ô3×9Ò9Ø��Q˜,ÐIÐI°ÐIÐI¸qÐIÐIÐIÔJÐPSÐTÑTÔTñVô Vð Vå�E”I˜| A¨¡E˜|˜|Ñ,Ô,Ô1×7Ò7Ø��Q˜,ÐGÐG°ÐGÐG¸qÐGÐGÐGÔHÈcÐRÑRÔRñTô Tð Tð Tñ	Tð%Tð TrU   úvisual.Ú
state_dictc                 ó  — i }g d¢}|                       ¦   «         D ]ê\  }}|                     |¦  «        sŒ|                     |d¦  «        }|D ]$}|                     |d         |d         ¦  «        }Œ%|dk    r;d}|                     dd¦  «        }t	          j        |j        d         ¦  «        |d<   nL|dk    r+d	}|                     d¦  «                             d¦  «        }n|d
k    r|                     d¦  «        }|||<   Œë|S )N)
)Úconv1úpatch_embed.proj)Úpositional_embeddingr7  )ztransformer.resblocks.rZ  )Úln_prero  )Úln_postr  )Úln_r  )Úin_proj_zqkv.©rµ   r}  )zmlp.c_fcúmlp.fc1)z
mlp.c_projzmlp.fc2r<   r   r0   r}  úhead.weightú	head.biasÚclass_embeddingrk  r7  )Úitemsrê  ÚreplacerÊ   r�   ÚzerosrÇ   rG  )rg  r  r‘  Úout_dictÚswapsrØ   rÙ   Úsps           rS   Ú_convert_openai_clipr{  ¥  s)  € ð
 €Hðð ð €Eð × Ò Ñ"Ô"ð ð ‰ˆˆ1Ø�|Š|˜FÑ#Ô#ð 	ØØ�IŠI�f˜bÑ!Ô!ˆØð 	(ð 	(ˆBØ—	’	˜"˜Qœ%  A¤Ñ'Ô'ˆAˆAà�Š;ˆ;ØˆAØ—’˜A˜qÑ!Ô!ˆAÝ$)¤K°´¸´
Ñ$;Ô$;ˆH�[Ñ!Ð!ØÐ#Ò#Ð#ØˆAØ—’˜A‘”×(Ò(¨Ñ+Ô+ˆAˆAØ�+ÒÐØ—’˜A‘”ˆAØˆ�‰ˆØ€OrU   c                 ó  — dd l }i }|                      dd ¦  «         d| v rg|                      d¦  «        |d<   |                      d¦  «        | d         d d …df         z   |d<   |                      d¦  «        d d …dd …f         |d<   |                      ¦   «         D ]j\  }}|                     d|¦  «        r|||                     d	d
¦  «        <   Œ5|                     d|¦  «        r|||                     dd¦  «        <   Œe|||<   Œk|S )Nr   Ú
mask_tokenÚregister_tokensrl  rk  r7  r0   z(blocks\.(\d+)\.mlp\.w12\.(?:weight|bias)Úw12Úfc1z'blocks\.(\d+)\.mlp\.w3\.(?:weight|bias)Úw3r  )ÚreÚpopru  Úmatchrv  )rg  r  r‚  rx  rØ   rÙ   s         rS   Ú_convert_dinov2r…  Ë  s4  € ð €I€I€IØ€HØ‡N‚N�< Ñ&Ô&Ð&Ø˜JÐ&Ð&à *§¢Ð/@Ñ AÔ Aˆ�ÑØ *§¢¨{Ñ ;Ô ;¸jÈÔ>UÐVWÐVWÐVWÐYZÐVZÔ>[Ñ [ˆ�ÑØ *§¢¨{Ñ ;Ô ;¸A¸A¸A¸q¸r¸r¸EÔ Bˆ�ÑØ× Ò Ñ"Ô"ð ð ‰ˆˆ1Ø�8Š8Ð?ÀÑCÔCð 	Ø01ˆH�Q—Y’Y˜u eÑ,Ô,Ñ-ØØ�XŠXÐ@À!ÑDÔDð 	Ø/0ˆH�Q—Y’Y˜t UÑ+Ô+Ñ,ØØˆ�‰ˆØ€OrU   c                 ó¨  — i }|                       ¦   «         D ]º\  }}|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     d	d
¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|||<   Œ»|S )NÚnorm_1rm   Únorm_2rr   zpreprocessor.patchifier.úpatch_embed.zpreprocessor.pos_embedr7  ztrunk.r<   zpost_trunk_norm.únorm.rq  z	mlp.fc1_gzmlp.fc3z	mlp.fc1_x)ru  rv  )rg  r  rx  rØ   rÙ   s        rS   Ú_convert_aimv2r‹  â  sÓ   € ð €HØ× Ò Ñ"Ô"ð 	ð 	‰ˆˆ1Ø�IŠI�h Ñ(Ô(ˆØ�IŠI�h Ñ(Ô(ˆØ�IŠIÐ0°.ÑAÔAˆØ�IŠIÐ.°Ñ<Ô<ˆØ�IŠI�h Ñ#Ô#ˆØ�IŠIÐ(¨'Ñ2Ô2ˆØ�IŠI�i Ñ-Ô-ˆØ�IŠI�i Ñ-Ô-ˆØˆ�‰ˆØ€OrU   c                 óè  — ddl }|                      d| ¦  «        } dD ]}|                      |d¦  «         Œg d¢}i }|                      ¦   «         D ]U\  }}d|v rŒ
|D ]\  }}|                     |||¦  «        }Œ|dk    r!|dd…                              d¦  «        |d	<   ŒP|||<   ŒVi i }
}	|                     d
¦  «        }|                     ¦   «         D ]ž\  }}|                     |¦  «        }|s||	|<   Œ"|                     ¦   «         \  }}}|
 	                    ||fi ¦  «        }|||<   t          |¦  «        dk    r4t          j        |d         |d         |d         gd¬¦  «        |	d|› d|› �<   ŒŸ|	S )zP
    Turn a BEiT-3 checkpoint into a standard VisionTransformer state-dict.
    r   Nr  )zbeit3.text_embed.weightzbeit3.vision_embed.mask_token))zbeit3\.r<   )zvision_embed\.cls_tokenrk  )zvision_embed\.r‰  )zembed_positions\.z
pos_embed.)z	encoder\.r<   )zlayers\.rZ  )zffn_layernorm\.rŠ  )zffn\.zmlp.)zself_attn_layer_norm\.znorm1.)zself_attn\.zattn.)zfinal_layer_norm\.znorm2.)Úinner_attn_lnr  rp  )z\.A\.ú.z.B.zpos_embed.weightrÂ   r7  z1blocks\.(\d+)\.attn\.(q|k|v)_proj\.(weight|bias)$r¥   r×   rØ   rÙ   rÁ   rZ  z
.attn.qkv.)r‚  rP   rƒ  ru  ÚsubrG  ÚcompileÚ	fullmatchÚgroupsÚ
setdefaultrÃ  r�   rÑ   )rg  r  r‚  rØ   ÚrulesÚtmprÙ   ÚoldÚnewÚoutÚbufÚpatrŒ  rÎ  ÚwhichÚkindÚstashs                    rS   Ú_convert_beit3rž  ô  sÚ  € ð €I€I€IØ—’ ¨Ñ4Ô4€Jð Jð  ð  ˆØ�Š�q˜$ÑÔÐÐðð ð €Eð  €CØ× Ò Ñ"Ô"ð 	ð 	‰ˆˆ1Ø�Aˆ:ˆ:ØØð 	$ð 	$‰HˆC�Ø—’�s˜C Ñ#Ô#ˆAˆAØÐ"Ò"Ð"à    œuŸš¨qÑ1Ô1ˆC�ÑÐàˆC�‰FˆFð �2ˆ€CØ
�*Š*ÐIÑ
JÔ
J€CØ—	’	‘”ð ð ‰ˆˆ1Ø�MŠM˜!ÑÔˆØð 	ØˆC�‰FØàŸ8š8™:œ:ÑˆˆU�DØ—’  T˜{¨BÑ/Ô/ˆØˆˆe‰Ýˆu‰:Œ:˜Š?ˆ?Ý38´9Ø�s”˜U 3œZ¨¨s¬Ð4¸!ð4ñ 4ô 4ˆCÐ/˜#Ð/Ð/¨Ð/Ð/Ñ0øð €JrU   Úadapt_layer_scalec           	      óÞ  ‡‡— ddl }i }|                      d| ¦  «        } |                      d| ¦  «        } dŠd| v rt          | |¦  «        } �n+d| v rt          | |d¬	¦  «        } �nd
| v rt          | |¦  «        } nþd| v r3dŠˆfd„|                      ¦   «         D ¦   «         } t          | |¦  «        } nÇt          d„ |                      ¦   «         D ¦   «         ¦  «        rt          | |¦  «        } n‹d| v r| d         } dŠn|d| v sd| v rYdŠd| v rRt          |j	        t          j        ¦  «        r3| d         |d<   t          j        | d         j        d         ¦  «        |d<   nd| v rdŠnd| v rt          | |¦  «        } ‰r ˆfd„|                      ¦   «         D ¦   «         } |                      ¦   «         D �]=\  }}d|v rŸ|j        j        j        j        \  }	}
}}t'          |j        ¦  «        dk     r3|j        j        j        j        \  }	}
}}|                     |	d||¦  «        }|j        d         |k    s|j        d         |k    rt+          |||f||d¬ ¦  «        }n�|d!k    re|j        d"         |j        j        d"         k    rDt/          |d#d$¦  «        rdnt/          |d%d"¦  «        }t1          ||j        j        |||d¬&¦  «        }n$|rd'|v r|                     d(d)|¦  «        }nd*|v r�Œ8|||<   �Œ?|S )+zJ convert patch embedding weight from manual patchify + linear proj to convr   Nr  rg  r<   zvisual.class_embeddingzmodule.visual.class_embeddingzmodule.visual.)r‘  r}  zvision_encoder.mask_tokenzvision_encoder.c                 óp   •— i | ]2\  }}|                      ‰¦  «        ¯|t          ‰¦  «        d …         |“Œ3S r–   ©rê  rÃ  )r  rØ   rÙ   Ú	ve_prefixs      €rS   ú
<dictcomp>z(checkpoint_filter_fn.<locals>.<dictcomp>G  s@   ø€ ÐfÐfÐf±°°1ÈaÏlÊlÐ[dÑNeÔNeÐf�a�˜I™œ˜˜Ô(¨!ÐfÐfÐfrU   c              3   ó   K  — | ]}d |v V — Œ	dS )zbeit3.Nrj   )r  rØ   s     rS   ú	<genexpr>z'checkpoint_filter_fn.<locals>.<genexpr>I  s&   è è € Ð6Ð6˜qˆX˜ˆ]Ð6Ð6Ð6Ð6Ð6Ð6rU   Úencoderzmodule.zvisual.trunk.pos_embedz"visual.trunk.blocks.0.norm1.weightzvisual.trunk.zvisual.head.proj.weightrr  rs  zmodule.visual.trunk.pos_embedzmodule.visual.trunk.z#preprocessor.patchifier.proj.weightc                 óp   •— i | ]2\  }}|                      ‰¦  «        ¯|t          ‰¦  «        d …         |“Œ3S r–   r¢  )r  rØ   rÙ   r‘  s      €rS   r¤  z(checkpoint_filter_fn.<locals>.<dictcomp>^  s?   ø€ Ð`Ð`Ð`©T¨Q°È1Ï<Ê<ÐX^ÑK_ÔK_Ð`�a�˜F™œ˜˜”o qÐ`Ð`Ð`rU   zpatch_embed.proj.weightr  rÀ   rÅ   Tr  r7  r0   r8  Fr  r'  Úgamma_zgamma_([0-9])z
ls\1.gammarÞ  )r‚  rP   r{  r…  ru  ÚanyÚkeysrž  rM   rt  rn   rª   r�   rw  rÇ   r‹  rh  r}  r™   rÃ  rÐ   r&   r7  rH  r'   r¨  r�  )rg  r  rŸ  rú  rû  r‚  rx  rØ   rÙ   ÚOÚIr¯  r°  r  r‘  r£  s                 @@rS   Úcheckpoint_filter_fnr®  0  s•  øø€ ð €I€I€IØ€HØ—’ ¨Ñ4Ô4€JØ—’ ¨jÑ9Ô9€JØ€Fà :Ð-Ð-Ý)¨*°eÑ<Ô<ˆ
‰
Ø	(¨JÐ	6Ð	6Ý)¨*°eÐDTÐUÑUÔUˆ
‰
Ø	˜Ð	#Ð	#Ý$ Z°Ñ7Ô7ˆ
ˆ
Ø	$¨
Ð	2Ð	2à%ˆ	ØfÐfÐfÐf¸
×8HÒ8HÑ8JÔ8JÐfÑfÔfˆ
Ý$ Z°Ñ7Ô7ˆ
ˆ
Ý	Ð6Ð6 J§O¢OÑ$5Ô$5Ð6Ñ6Ô6Ñ	6Ô	6ð 7å# J°Ñ6Ô6ˆ
ˆ
Ø	�jÐ	 Ð	 à 	Ô*ˆ
ØˆˆØ	! ZÐ	/Ð	/Ð3WÐ[eÐ3eÐ3eà ˆØ$¨
Ð2Ð2µzÀ%Ä*ÍbÌiÑ7XÔ7XÐ2à&0Ð1JÔ&KˆH�]Ñ#Ý$)¤K°
Ð;TÔ0UÔ0[Ð\]Ô0^Ñ$_Ô$_ˆH�[Ñ!øØ	(¨JÐ	6Ð	6Ø'ˆˆØ	.°*Ð	<Ð	<Ý# J°Ñ6Ô6ˆ
àð aà`Ð`Ð`Ð`°Z×5EÒ5EÑ5GÔ5GÐ`Ñ`Ô`ˆ
à× Ò Ñ"Ô"ð  ñ  ‰ˆˆ1Ø$¨Ð)Ð)ØÔ*Ô/Ô6Ô<‰JˆAˆq�!�QÝ�1”7‰|Œ|˜aÒÐà"Ô.Ô3Ô:Ô@‘
��1�a˜Ø—I’I˜a  Q¨Ñ*Ô*�ØŒw�rŒ{˜aÒÐ 1¤7¨2¤;°!Ò#3Ð#3Ý(ØØ˜�FØ"/Ø'Ø ðñ ô �øð �+ÒÐ !¤'¨!¤*°´Ô0EÀaÔ0HÒ"HÐ"Hå%,¨UÐ4DÀeÑ%LÔ%LÐ x  ÕRYÐZ_ÐatÐvwÑRxÔRxÐÝ&ØØÔ*Ô4Ø"3Ø+Ø#Øðñ ô ˆAˆAð ð 	 8¨q = =à—’Ð'¨¸Ñ:Ô:ˆAˆAØ˜QÐÐáØˆ�‰‰Ø€OrU   Úurlc                 ó8   — | ddd dddt           t          dddd	œ|¥S )
Nr)  )r¥   r'  r'  gÍÌÌÌÌÌì?rö  Trj  rt  ú
apache-2.0)r¯  r0  Ú
input_sizeÚ	pool_sizeÚcrop_pctrú  Úfixed_input_sizerõ   rö   Ú
first_convÚ
classifierÚlicense)r   r   )r¯  rR   s     rS   Ú_cfgr¹  „  s?   € àØØ#ØØØ"Ø Ý'Ý%Ø(ØØðð ð ðð rU   z*vit_base_patch16_224.augreg2_in21k_ft_in1kztimm/)Ú	hf_hub_idz)vit_base_patch8_224.augreg2_in21k_ft_in1kz)vit_tiny_patch16_224.augreg_in21k_ft_in1kz•https://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npz)r¯  rº  Úcustom_loadz)vit_tiny_patch16_384.augreg_in21k_ft_in1kz•https://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)r¥   é€  r¼  ç      ð?)r¯  rº  r»  r²  r´  z*vit_small_patch32_224.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz*vit_small_patch32_384.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz*vit_small_patch16_224.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz*vit_small_patch16_384.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz)vit_base_patch32_224.augreg_in21k_ft_in1kz—https://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz)vit_base_patch32_384.augreg_in21k_ft_in1kz•https://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_light1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz)vit_base_patch16_224.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz)vit_base_patch16_384.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz(vit_base_patch8_224.augreg_in21k_ft_in1kz•https://storage.googleapis.com/vit_models/augreg/B_8-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz*vit_large_patch16_224.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz*vit_large_patch16_384.augreg_in21k_ft_in1kz–https://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzz'vit_base_patch16_224.orig_in21k_ft_in1kzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_p16_224-80ecf9dd.pth)r¯  rº  z'vit_base_patch16_384.orig_in21k_ft_in1kzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_p16_384-83fb41ba.pth)r¯  rº  r²  r´  z(vit_large_patch32_384.orig_in21k_ft_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_large_p32_384-9b920ba8.pthz!vit_small_patch16_224.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/S_16-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz!vit_small_patch16_384.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/S_16-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz vit_base_patch32_224.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/B_32-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz vit_base_patch32_384.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/B_32-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzz vit_base_patch16_224.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/B_16-i1k-300ep-lr_0.001-aug_strong2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz vit_base_patch16_384.augreg_in1kz•https://storage.googleapis.com/vit_models/augreg/B_16-i1k-300ep-lr_0.001-aug_strong2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzzvit_large_patch14_224.untrained)r¯  zvit_huge_patch14_224.untrainedzvit_giant_patch14_224.untrainedz"vit_gigantic_patch14_224.untrainedzvit_base_patch32_224.orig_in21k)rº  r0  zvit_base_patch16_224.orig_in21kz vit_large_patch32_224.orig_in21kz vit_large_patch16_224.orig_in21kzvit_huge_patch14_224.orig_in21kz!vit_tiny_patch16_224.augreg_in21kzmhttps://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0.npziSU  )r¯  rº  r»  r0  z"vit_small_patch32_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0.npzz"vit_small_patch16_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0.npzz!vit_base_patch32_224.augreg_in21kzohttps://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.03-do_0.0-sd_0.0.npzz!vit_base_patch16_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npzz vit_base_patch8_224.augreg_in21kzmhttps://storage.googleapis.com/vit_models/augreg/B_8-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npzz"vit_large_patch16_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1.npzzvit_base_patch32_224.sam_in1kz:https://storage.googleapis.com/vit_models/sam/ViT-B_32.npz)r¯  r»  rº  zvit_base_patch16_224.sam_in1kz:https://storage.googleapis.com/vit_models/sam/ViT-B_16.npzzvit_small_patch16_224.dinoz[https://dl.fbaipublicfiles.com/dino/dino_deitsmall16_pretrain/dino_deitsmall16_pretrain.pth)r¯  rº  rõ   rö   r0  zvit_small_patch8_224.dinozYhttps://dl.fbaipublicfiles.com/dino/dino_deitsmall8_pretrain/dino_deitsmall8_pretrain.pthzvit_base_patch16_224.dinozWhttps://dl.fbaipublicfiles.com/dino/dino_vitbase16_pretrain/dino_vitbase16_pretrain.pthzvit_base_patch8_224.dinozUhttps://dl.fbaipublicfiles.com/dino/dino_vitbase8_pretrain/dino_vitbase8_pretrain.pthz vit_small_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_pretrain.pthr±  )r¥   é  r¾  )r¯  rº  r¸  rõ   rö   r0  r²  r´  zvit_base_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_pretrain.pthz vit_large_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pthz vit_giant_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_pretrain.pthz%vit_small_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_reg4_pretrain.pthz$vit_base_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_reg4_pretrain.pthz%vit_large_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_reg4_pretrain.pthz%vit_giant_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_reg4_pretrain.pthz"vit_base_patch14_reg1_tipsv2.webli)r?   r?   r?   )r½  r½  r½  )r¥   éÀ  r¿  )rº  r¸  rõ   rö   r0  r²  r´  z#vit_large_patch14_reg1_tipsv2.webliz$vit_so400m_patch14_reg1_tipsv2.webliz#vit_giant_patch14_reg1_tipsv2.weblizvit_base_patch16_224_miil.in21kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/vit_base_patch16_224_in21k_miil-887286df.pthg      ì?r  iÕ+  )r¯  rº  rõ   rö   r´  rú  r0  z'vit_base_patch16_224_miil.in21k_ft_in1kzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/vit_base_patch16_224_1k_miil_84_4-2deb18e3.pth)r¯  rº  rõ   rö   r´  rú  z vit_base_patch16_rpn_224.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_base_patch16_rpn_224-sw-3b07e89d.pthz#vit_medium_patch16_gap_240.sw_in12k)r¥   éð   rÀ  gffffffî?i-.  )rº  r²  r´  r0  z+vit_medium_patch16_gap_256.sw_in12k_ft_in1k)r¥   é   rÁ  )rº  r²  r´  z+vit_medium_patch16_gap_384.sw_in12k_ft_in1kÚsquash)rº  r²  r´  Ú	crop_modez%vit_betwixt_patch16_gap_256.untrained)r²  r´  z"vit_base_patch16_gap_224.untrainedz/vit_base_patch32_clip_224.laion2b_ft_in12k_in1k)rº  rõ   rö   z/vit_base_patch32_clip_384.laion2b_ft_in12k_in1k)rº  rõ   rö   r´  r²  z/vit_base_patch32_clip_448.laion2b_ft_in12k_in1kz/vit_base_patch16_clip_224.laion2b_ft_in12k_in1k)rº  rõ   rö   r´  z/vit_base_patch16_clip_384.laion2b_ft_in12k_in1k)rº  rõ   rö   r´  r²  rÃ  z0vit_large_patch14_clip_224.laion2b_ft_in12k_in1kz0vit_large_patch14_clip_336.laion2b_ft_in12k_in1k)r¥   éP  rÄ  z/vit_huge_patch14_clip_224.laion2b_ft_in12k_in1kz/vit_huge_patch14_clip_336.laion2b_ft_in12k_in1kz.vit_base_patch32_clip_224.openai_ft_in12k_in1krô   z.vit_base_patch32_clip_384.openai_ft_in12k_in1kz.vit_base_patch16_clip_224.openai_ft_in12k_in1kz.vit_base_patch16_clip_384.openai_ft_in12k_in1kz/vit_large_patch14_clip_224.openai_ft_in12k_in1kz/vit_large_patch14_clip_336.openai_ft_in12k_in1kz)vit_base_patch32_clip_224.laion2b_ft_in1kz)vit_base_patch16_clip_224.laion2b_ft_in1kz)vit_base_patch16_clip_384.laion2b_ft_in1kz*vit_large_patch14_clip_224.laion2b_ft_in1kz*vit_large_patch14_clip_336.laion2b_ft_in1kz)vit_huge_patch14_clip_224.laion2b_ft_in1kz)vit_huge_patch14_clip_336.laion2b_ft_in1kz(vit_base_patch32_clip_224.openai_ft_in1kz(vit_base_patch16_clip_224.openai_ft_in1kz(vit_base_patch16_clip_384.openai_ft_in1kz)vit_large_patch14_clip_224.openai_ft_in1kz*vit_base_patch16_clip_224.laion2b_ft_in12k)rº  rõ   rö   r0  z+vit_large_patch14_clip_224.laion2b_ft_in12k)rº  rõ   rö   r´  r0  z*vit_huge_patch14_clip_224.laion2b_ft_in12kz)vit_base_patch16_clip_224.openai_ft_in12kz*vit_large_patch14_clip_224.openai_ft_in12kz!vit_base_patch32_clip_224.laion2bé   z!vit_base_patch16_clip_224.laion2bz"vit_large_patch14_clip_224.laion2br*  z!vit_huge_patch14_clip_224.laion2bé   z"vit_giant_patch14_clip_224.laion2bz%vit_gigantic_patch14_clip_224.laion2bé   z'vit_base_patch32_clip_224.laion400m_e32Úmit)zDnatively QuickGELU, use quickgelu model variant for original results)rº  r¸  Únotesrõ   rö   r0  z'vit_base_patch16_clip_224.laion400m_e32)rº  r¸  rõ   rö   r´  r0  z,vit_base_patch16_plus_clip_240.laion400m_e32é€  )rº  r¸  rõ   rö   r²  r´  r0  z(vit_large_patch14_clip_224.laion400m_e32z$vit_base_patch32_clip_224.datacompxlz$vit_base_patch32_clip_256.datacompxl)rº  rõ   rö   r´  r²  r0  z$vit_base_patch16_clip_224.datacompxlz%vit_large_patch14_clip_224.datacompxlzvit_base_patch16_clip_224.dfn2bz
apple-asclz%vit_large_patch14_clip_224.dfn2b_s39bz vit_large_patch14_clip_224.dfn2b)rº  r¸  rÉ  rõ   rö   r´  r0  zvit_huge_patch14_clip_224.dfn5bzvit_huge_patch14_clip_378.dfn5b)r¥   éz  rË  )rº  rõ   rö   r¸  rÉ  r´  r²  r0  z-vit_huge_patch14_clip_224.metaclip2_worldwidezcc-by-nc-4.0z-vit_huge_patch14_clip_378.metaclip2_worldwide)rº  r¸  rõ   rö   r²  r´  rÃ  r0  z1vit_gigantic_patch14_clip_224.metaclip2_worldwidez1vit_gigantic_patch14_clip_378.metaclip2_worldwidez(vit_base_patch32_clip_224.metaclip_2pt5bz(vit_base_patch16_clip_224.metaclip_2pt5bz)vit_large_patch14_clip_224.metaclip_2pt5bz(vit_huge_patch14_clip_224.metaclip_2pt5bz-vit_huge_patch14_clip_224.metaclip_altogetherz,vit_gigantic_patch14_clip_224.metaclip_2pt5bz'vit_base_patch32_clip_224.metaclip_400mz'vit_base_patch16_clip_224.metaclip_400mz(vit_large_patch14_clip_224.metaclip_400mz vit_base_patch32_clip_224.openai)rº  rÉ  rõ   rö   r0  z vit_base_patch16_clip_224.openaiz!vit_large_patch14_clip_224.openai)rº  rÉ  rõ   rö   r´  r0  z!vit_large_patch14_clip_336.openai)rº  rÉ  rõ   rö   r´  r²  r0  z/vit_large_patch14_clip_224.apple_mclip2_dfndr2bz
apple-amlr)rº  r0  rõ   rö   r´  r¸  z#vit_base_patch32_plus_256.untrained)r¯  r²  r´  z#vit_base_patch16_plus_240.untrainedz$vit_small_patch16_36x1_224.untrainedz$vit_small_patch16_18x2_224.untrainedz#vit_base_patch16_18x2_224.untrainedz)eva_large_patch14_196.in22k_ft_in22k_in1k)r¥   éÄ   rÌ  )rº  r¸  rõ   rö   r²  r´  z)eva_large_patch14_336.in22k_ft_in22k_in1k)rº  r¸  rõ   rö   r²  r´  rÃ  z#eva_large_patch14_196.in22k_ft_in1kz#eva_large_patch14_336.in22k_ft_in1kzflexivit_small.1200ep_in1kzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k.npz)r¯  r»  rº  r²  r´  zflexivit_small.600ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k_600ep.npzzflexivit_small.300ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k_300ep.npzzflexivit_base.1200ep_in1kzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k.npzzflexivit_base.600ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k_600ep.npzzflexivit_base.300ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k_300ep.npzzflexivit_base.1000ep_in21kzMhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i21k_1000ep.npz)r¯  r»  rº  r²  r´  r0  zflexivit_base.300ep_in21kzLhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i21k_300ep.npzzflexivit_large.1200ep_in1kzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k.npzzflexivit_large.600ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k_600ep.npzzflexivit_large.300ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k_300ep.npzzflexivit_base.patch16_in21kzIhttps://storage.googleapis.com/big_vision/flexivit/vit_b16_i21k_300ep.npzzflexivit_base.patch30_in21kzIhttps://storage.googleapis.com/big_vision/flexivit/vit_b30_i21k_300ep.npzz!vit_base_patch16_xp_224.untrainedz"vit_large_patch14_xp_224.untrainedz!vit_huge_patch14_xp_224.untrainedzvit_base_patch16_224.maezEhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_base.pth)r¯  rº  r¸  rõ   rö   r0  zvit_large_patch16_224.maezFhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_large.pthzvit_huge_patch14_224.maezEhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_huge.pthz#vit_huge_patch14_gap_224.in1k_ijepaz?https://dl.fbaipublicfiles.com/ijepa/IN1K-vit.h.14-300e.pth.tar)r¯  r¸  rõ   rö   r0  z$vit_huge_patch14_gap_224.in22k_ijepaz@https://dl.fbaipublicfiles.com/ijepa/IN22K-vit.h.14-900e.pth.tarz#vit_huge_patch16_gap_448.in1k_ijepazEhttps://dl.fbaipublicfiles.com/ijepa/IN1K-vit.h.16-448px-300e.pth.tar)r¯  r¸  r²  r´  rõ   rö   r0  z%vit_giant_patch16_gap_224.in22k_ijepaz@https://dl.fbaipublicfiles.com/ijepa/IN22K-vit.g.16-600e.pth.tarz$vit_base_patch32_siglip_256.v2_webli)rº  r²  r0  z$vit_base_patch16_siglip_224.v2_webliz!vit_base_patch16_siglip_224.webliz$vit_base_patch16_siglip_256.v2_webliz!vit_base_patch16_siglip_256.webliz&vit_base_patch16_siglip_256.webli_i18nz$vit_base_patch16_siglip_384.v2_webliz!vit_base_patch16_siglip_384.webliz$vit_base_patch16_siglip_512.v2_webli)r¥   rÅ  rÅ  z!vit_base_patch16_siglip_512.webliz%vit_large_patch16_siglip_256.v2_webliz"vit_large_patch16_siglip_256.webliz%vit_large_patch16_siglip_384.v2_webliz"vit_large_patch16_siglip_384.webliz%vit_large_patch16_siglip_512.v2_webliz&vit_so400m_patch14_siglip_224.v2_webliz#vit_so400m_patch14_siglip_224.webliz&vit_so400m_patch14_siglip_378.v2_webliz#vit_so400m_patch14_siglip_378.webliz#vit_so400m_patch14_siglip_384.webliz&vit_so400m_patch16_siglip_256.v2_webliz(vit_so400m_patch16_siglip_256.webli_i18nz&vit_so400m_patch16_siglip_384.v2_webliz&vit_so400m_patch16_siglip_512.v2_webliz(vit_giantopt_patch16_siglip_256.v2_webliz(vit_giantopt_patch16_siglip_384.v2_webliz(vit_base_patch32_siglip_gap_256.v2_webliz(vit_base_patch16_siglip_gap_224.v2_webliz%vit_base_patch16_siglip_gap_224.webliz(vit_base_patch16_siglip_gap_256.v2_webliz%vit_base_patch16_siglip_gap_256.webliz*vit_base_patch16_siglip_gap_256.webli_i18nz(vit_base_patch16_siglip_gap_384.v2_webliz%vit_base_patch16_siglip_gap_384.webliz(vit_base_patch16_siglip_gap_512.v2_webliz%vit_base_patch16_siglip_gap_512.webliz)vit_large_patch16_siglip_gap_256.v2_webliz&vit_large_patch16_siglip_gap_256.webliz)vit_large_patch16_siglip_gap_384.v2_webliz&vit_large_patch16_siglip_gap_384.webliz)vit_large_patch16_siglip_gap_512.v2_webliz*vit_so400m_patch14_siglip_gap_224.v2_webliz'vit_so400m_patch14_siglip_gap_224.webliz*vit_so400m_patch14_siglip_gap_224.pali_mixz)vit_so400m_patch14_siglip_gap_224.pali_ptz-vit_so400m_patch14_siglip_gap_224.pali2_3b_ptz.vit_so400m_patch14_siglip_gap_224.pali2_10b_ptz*vit_so400m_patch14_siglip_gap_378.v2_webliz'vit_so400m_patch14_siglip_gap_378.webliz'vit_so400m_patch14_siglip_gap_384.webliz*vit_so400m_patch14_siglip_gap_448.pali_mixz)vit_so400m_patch14_siglip_gap_448.pali_ptz2vit_so400m_patch14_siglip_gap_448.pali_refcoco_segz-vit_so400m_patch14_siglip_gap_448.pali_ocrvqaz-vit_so400m_patch14_siglip_gap_448.pali2_3b_ptz.vit_so400m_patch14_siglip_gap_448.pali2_10b_ptz0vit_so400m_patch14_siglip_gap_448.pali2_3b_docciz1vit_so400m_patch14_siglip_gap_448.pali2_10b_docciz)vit_so400m_patch14_siglip_gap_896.pali_pt)r¥   é€  rÍ  z2vit_so400m_patch14_siglip_gap_896.pali_refcoco_segz-vit_so400m_patch14_siglip_gap_896.pali_ocrvqaz-vit_so400m_patch14_siglip_gap_896.pali2_3b_ptz.vit_so400m_patch14_siglip_gap_896.pali2_10b_ptz*vit_so400m_patch16_siglip_gap_256.v2_webliz,vit_so400m_patch16_siglip_gap_256.webli_i18nz*vit_so400m_patch16_siglip_gap_384.v2_webliz*vit_so400m_patch16_siglip_gap_512.v2_webliz,vit_giantopt_patch16_siglip_gap_256.v2_webliz,vit_giantopt_patch16_siglip_gap_384.v2_webliz+vit_so400m_patch14_siglip_378.webli_ft_in1kz/vit_so400m_patch14_siglip_gap_378.webli_ft_in1kz,vit_xsmall_patch16_clip_224.tinyclip_yfcc15m)rº  r¸  rõ   rö   r0  z.vit_medium_patch32_clip_224.tinyclip_laion400mz,vit_medium_patch16_clip_224.tinyclip_yfcc15mz/vit_betwixt_patch32_clip_224.tinyclip_laion400mz%vit_wee_patch16_reg1_gap_256.sbb_in1kz/vit_dwee_patch16_reg1_gap_256.sbb_nadamuon_in1kz&vit_dwee_patch16_reg1_gap_256.sbb_in1kz&vit_pwee_patch16_reg1_gap_256.sbb_in1kz0vit_dpwee_patch16_reg1_gap_256.sbb_nadamuon_in1kz'vit_dpwee_patch16_reg1_gap_256.sbb_in1kz1vit_little_patch16_reg1_gap_256.sbb_in12k_ft_in1kz)vit_little_patch16_reg1_gap_256.sbb_in12k)rº  r0  r²  r´  z(vit_little_patch16_reg4_gap_256.sbb_in1kz2vit_dlittle_patch16_reg1_gap_256.sbb_nadamuon_in1kz(vit_medium_patch16_reg1_gap_256.sbb_in1kz1vit_medium_patch16_reg4_gap_256.sbb_in12k_ft_in1kz(vit_medium_patch16_reg4_gap_256.sbb_in1kz)vit_medium_patch16_reg4_gap_256.sbb_in12kz8vit_mediumd_patch16_reg4_gap_256.sbb2_e200_in12k_ft_in1kz2vit_mediumd_patch16_reg4_gap_256.sbb_in12k_ft_in1kz0vit_mediumd_patch16_reg4_gap_256.sbb2_e200_in12kz*vit_mediumd_patch16_reg4_gap_256.sbb_in12kz8vit_mediumd_patch16_reg4_gap_384.sbb2_e200_in12k_ft_in1kz)vit_betwixt_patch16_reg1_gap_256.sbb_in1kz8vit_betwixt_patch16_reg4_gap_256.sbb2_e200_in12k_ft_in1kz2vit_betwixt_patch16_reg4_gap_256.sbb_in12k_ft_in1kz)vit_betwixt_patch16_reg4_gap_256.sbb_in1kz0vit_betwixt_patch16_reg4_gap_256.sbb2_e200_in12kz*vit_betwixt_patch16_reg4_gap_256.sbb_in12kz8vit_betwixt_patch16_reg4_gap_384.sbb2_e200_in12k_ft_in1kz'vit_base_patch16_reg4_gap_256.untrained)r²  z6vit_so150m_patch16_reg4_gap_256.sbb_e250_in12k_ft_in1kz.vit_so150m_patch16_reg4_gap_256.sbb_e250_in12kz6vit_so150m_patch16_reg4_gap_384.sbb_e250_in12k_ft_in1kz)vit_so150m_patch16_reg4_map_256.untrainedz7vit_so150m2_patch16_reg1_gap_256.sbb_e200_in12k_ft_in1kz/vit_so150m2_patch16_reg1_gap_256.sbb_e200_in12kz7vit_so150m2_patch16_reg1_gap_384.sbb_e200_in12k_ft_in1kz7vit_so150m2_patch16_reg1_gap_448.sbb_e200_in12k_ft_in1kz$vit_intern300m_patch14_448.ogvl_distz$vit_intern300m_patch14_448.ogvl_2pt5)rº  rõ   rö   r²  r´  r0  z aimv2_large_patch14_224.apple_pt)rº  rõ   rö   r¸  r´  r0  z%aimv2_large_patch14_224.apple_pt_distzaimv2_huge_patch14_224.apple_ptzaimv2_1b_patch14_224.apple_ptzaimv2_3b_patch14_224.apple_ptz aimv2_large_patch14_336.apple_pt)rº  rõ   rö   r¸  r²  r´  r0  z%aimv2_large_patch14_336.apple_pt_distzaimv2_huge_patch14_336.apple_ptzaimv2_1b_patch14_336.apple_ptzaimv2_3b_patch14_336.apple_ptz aimv2_large_patch14_448.apple_ptzaimv2_huge_patch14_448.apple_ptzaimv2_1b_patch14_448.apple_ptzaimv2_3b_patch14_448.apple_ptztest_vit.r160_in1k)r¥   é    rÎ  ztest_vit2.r160_in1kztest_vit3.r160_in1kztest_vit4.r160_in1kz$beit3_base_patch16_224.in22k_ft_in1kz-beit3_base_patch16_224.indomain_in22k_ft_in1kz%beit3_large_patch16_224.in22k_ft_in1kz.beit3_large_patch16_224.indomain_in22k_ft_in1kz!beit3_giant_patch14_224.untrained)r¯  rõ   rö   r´  z!beit3_giant_patch14_336.untrained)r¯  r²  rõ   rö   r´  zbeit3_base_patch16_224.ptz"beit3_base_patch16_224.indomain_ptzbeit3_large_patch16_224.ptz#beit3_large_patch16_224.indomain_ptc                 ód   — g | ]-\  }}|                      d d¦  «        ¯d|d          d         v ¯+|‘Œ.S )rÉ  rj   Ú	quickgelur   )rP   )r  r×  Úcs      rS   r  r  j  sG   € ÐnÐnÐn™$˜!˜Q¸!¿%º%ÀÈÑ:LÔ:LÐnÐQ\Ð`aÐbiÔ`jÐklÔ`mÐQmÐQm�AÐQmÐQmÐQmrU   rº  Ú_clip_Ú_clip_quickgelu_ÚTIMM_USE_NAFLEXVITÚfalseÚtrueÚvariantÚ
pretrainedÚ
use_naflexÚ	NaFlexVitc           	      ód  — |€t           }|rddlm}  || |fi |¤ŽS |                     dd¦  «        }d| v rt	          t
          dd¬¦  «        }nt
          }|                     d	d
¦  «        }d| v r|                     dd ¦  «        dk    rd}t          t          | |f||t          |d¬¦  «        dœ|¤ŽS )Nr0   )Ú_create_naflexvit_from_classicÚout_indicesr¥   Úflexir  F)rú  rû  Úpretrained_strictTÚsiglipr1  r3  Úgetter)rÝ  Úfeature_cls)Úpretrained_filter_fnrß  Úfeature_cfg)
Ú_USE_NAFLEX_DEFAULTÚ	naflexvitrÜ  rƒ  r   r®  rP   r1   r;   r_  )r×  rØ  rÙ  rR   rÜ  rÝ  Ú
_filter_fnÚstricts           rS   Ú_create_vision_transformerré  w  s  € ð ÐÝ(ˆ
Øð Mà=Ð=Ð=Ð=Ð=Ð=Ø-Ð-¨g°zÐLÐLÀVÐLÐLÐLà—*’*˜]¨AÑ.Ô.€KØ�'ÐÐõ Õ1ÀÐW\Ð]Ñ]Ô]ˆ
ˆ
å)ˆ
ð �ZŠZÐ+¨TÑ2Ô2€FØ�7ÐÐ˜vŸzšz¨-¸Ñ>Ô>À%ÒGÐGØˆåÝØØðð (Ø Ý [¸hÐGÑGÔGðð ð ðð ð rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	z ViT-Tiny (Vit-Ti/16)
    r(  éÀ   r+  r¥   ©r.  r5  rJ   rB   Úvit_tiny_patch16_224rØ  )rí  ©r_  ré  ©rØ  rR   Ú
model_argsr  s       rS   rí  rí  �  óJ   € õ  ¨s¸"ÈÐJÑJÔJ€JÝ&ÐsÐsÈ*ÐsÕX\Ð]gÐXrÐXrÐkqÐXrÐXrÐsÐs€EØ€LrU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	z% ViT-Tiny (Vit-Ti/16) @ 384x384.
    r(  rë  r+  r¥   rì  Úvit_tiny_patch16_384rØ  )ró  rî  rï  s       rS   ró  ró  ¦  rñ  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	z ViT-Small (ViT-S/32)
    é    r¼  r+  é   rì  Úvit_small_patch32_224rØ  )r÷  rî  rï  s       rS   r÷  r÷  ¯  óJ   € õ  ¨s¸"ÈÐJÑJÔJ€JÝ&ÐtÐtÈ:ÐtÕY]Ð^hÐYsÐYsÐlrÐYsÐYsÐtÐt€EØ€LrU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	z& ViT-Small (ViT-S/32) at 384x384.
    rõ  r¼  r+  rö  rì  Úvit_small_patch32_384rØ  )rú  rî  rï  s       rS   rú  rú  ¸  rø  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	ú ViT-Small (ViT-S/16)
    r(  r¼  r+  rö  rì  Úvit_small_patch16_224rØ  )rý  rî  rï  s       rS   rý  rý  Á  rø  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	rü  r(  r¼  r+  rö  rì  Úvit_small_patch16_384rØ  )rÿ  rî  rï  s       rS   rÿ  rÿ  Ê  rø  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	z ViT-Small (ViT-S/8)
    é   r¼  r+  rö  rì  Úvit_small_patch8_224rØ  )r  rî  rï  s       rS   r  r  Ó  sJ   € õ  ¨c¸ÀqÐIÑIÔI€JÝ&ÐsÐsÈ*ÐsÕX\Ð]gÐXrÐXrÐkqÐXrÐXrÐsÐs€EØ€LrU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )z¾ ViT-Base (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k, source https://github.com/google-research/vision_transformer.
    rõ  r*  r+  rì  Úvit_base_patch32_224rØ  )r  rî  rï  s       rS   r  r  Ü  óJ   € õ
  ¨s¸"ÈÐKÑKÔK€JÝ&ÐsÐsÈ*ÐsÕX\Ð]gÐXrÐXrÐkqÐXrÐXrÐsÐs€EØ€LrU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )zÎ ViT-Base model (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    rõ  r*  r+  rì  Úvit_base_patch32_384rØ  )r  rî  rï  s       rS   r  r  æ  r  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )zÈ ViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    r(  r*  r+  rì  Úvit_base_patch16_224rØ  )r	  rî  rï  s       rS   r	  r	  ð  r  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )zÎ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r(  r*  r+  rì  Úvit_base_patch16_384rØ  )r  rî  rï  s       rS   r  r  ú  r  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )zÇ ViT-Base (ViT-B/8) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    r  r*  r+  rì  Úvit_base_patch8_224rØ  )r  rî  rï  s       rS   r  r    sJ   € õ
  ¨c¸ÀrÐJÑJÔJ€JÝ&ÐrÐrÈÐrÕW[Ð\fÐWqÐWqÐjpÐWqÐWqÐrÐr€EØ€LrU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	zo ViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929). No pretrained weights.
    rõ  rÆ  é   r(  rì  Úvit_large_patch32_224rØ  )r  rî  rï  s       rS   r  r    óJ   € õ  ¨t¸2ÈÐLÑLÔL€JÝ&ÐtÐtÈ:ÐtÕY]Ð^hÐYsÐYsÐlrÐYsÐYsÐtÐt€EØ€LrU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	zÏ ViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    rõ  rÆ  r  r(  rì  Úvit_large_patch32_384rØ  )r  rî  rï  s       rS   r  r    óJ   € õ
  ¨t¸2ÈÐLÑLÔL€JÝ&ÐtÐtÈ:ÐtÕY]Ð^hÐYsÐYsÐlrÐYsÐYsÐtÐt€EØ€LrU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )zÏ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    r(  rÆ  r  rì  Úvit_large_patch16_224rØ  )r  rî  rï  s       rS   r  r  !  r  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )zÏ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r(  rÆ  r  rì  Úvit_large_patch16_384rØ  )r  rî  rï  s       rS   r  r  +  r  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	z  ViT-Large model (ViT-L/14)
    é   rÆ  r  r(  rì  Úvit_large_patch14_224rØ  )r  rî  rï  s       rS   r  r  5  r  rU   c           	      ó^   — t          dddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	zW ViT-Huge model (ViT-H/14) from original paper (https://arxiv.org/abs/2010.11929).
    r  rÇ  rõ  r(  rì  Úvit_huge_patch14_224rØ  )r  rî  rï  s       rS   r  r  >  sJ   € õ  ¨t¸2ÈÐLÑLÔL€JÝ&ÐsÐsÈ*ÐsÕX\Ð]gÐXrÐXrÐkqÐXrÐXrÐsÐs€EØ€LrU   c           	      ó`   — t          ddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
zq ViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    r  é€  çtÑE]t@é(   r(  ©r.  r5  rY   rJ   rB   Úvit_giant_patch14_224rØ  )r#  rî  rï  s       rS   r#  r#  G  sM   € õ  ¨t¸uÈBÐZ\Ð]Ñ]Ô]€JÝ&ÐtÐtÈ:ÐtÕY]Ð^hÐYsÐYsÐlrÐYsÐYsÐtÐt€EØ€LrU   c           	      ób   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
zq ViT-Gigantic (big-G) model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    r  é€  ç;±;±@é0   r(  r"  Úvit_gigantic_patch14_224rØ  )r(  rî  rï  s       rS   r(  r(  P  s_   € õ  ¨t¸uÈBÐZ\Ð]Ñ]Ô]€JÝ&Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Y€Eà€LrU   c           	      ób   — t          ddddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )	z• ViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    Weights taken from: https://github.com/Alibaba-MIIL/ImageNet21K
    r(  r*  r+  F)r.  r5  rJ   rB   rC   Úvit_base_patch16_224_miilrØ  )r*  rî  rï  s       rS   r*  r*  Z  s_   € õ
  ¨s¸"ÈÐUZÐ[Ñ[Ô[€JÝ&Ø#ðZð ZØ0:ðZÝ>BÀ:Ð>XÐ>XÐQWÐ>XÐ>XðZð Z€Eà€LrU   c                 ój   — t          ddddddddd¬¦	  «	        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 240x240
    r(  rÅ  r+  r  Fr   rˆ  ©	r.  r5  rJ   rB   r6  r1  rC   r\   r<  Úvit_medium_patch16_gap_240rØ  )r-  rî  rï  s       rS   r-  r-  e  ós   € õ Ø ¨B¸!ÈØ E°tÀUðLñ Lô L€Jõ 'Ø$ð[ð [Ø1;ð[Ý?CÀJÐ?YÐ?YÐRXÐ?YÐ?Yð[ð [€Eà€LrU   c                 ój   — t          ddddddddd¬¦	  «	        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 256x256
    r(  rÅ  r+  r  Fr   rˆ  r,  Úvit_medium_patch16_gap_256rØ  )r0  rî  rï  s       rS   r0  r0  q  r.  rU   c                 ój   — t          ddddddddd¬¦	  «	        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 384x384
    r(  rÅ  r+  r  Fr   rˆ  r,  Úvit_medium_patch16_gap_384rØ  )r2  rî  rï  s       rS   r2  r2  }  r.  rU   c                 ój   — t          ddddddddd¬¦	  «	        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )zC ViT-Betwixt (ViT-b/16) w/o class token, w/ avg-pool @ 256x256
    r(  rÊ  r+  é
   Fr   rˆ  r,  Úvit_betwixt_patch16_gap_256rØ  )r5  rî  rï  s       rS   r5  r5  ‰  ss   € õ Ø ¨B¸"È%Ø E°tÀUðLñ Lô L€Jõ 'Ø%ð\ð \Ø2<ð\Ý@DÀZÐ@ZÐ@ZÐSYÐ@ZÐ@Zð\ð \€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
z@ ViT-Base (ViT-B/16) w/o class token, w/ avg-pool @ 224x224
    r(  r*  r+  Fr   ©r.  r5  rJ   rB   r6  r1  r<  Úvit_base_patch16_gap_224rØ  )r8  rî  rï  s       rS   r8  r8  •  sn   € õ Ø ¨B¸"È%Ð]bÐlqðsñ sô s€Jå&Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Y€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )z; ViT-Huge model (ViT-H/14) w/ no class token, avg pool
    r  rÇ  rõ  r(  Fr   r7  Úvit_huge_patch14_gap_224rØ  )r:  rî  rï  s       rS   r:  r:     ón   € õ Ø ¨R¸2È5Ð^cÐmrðtñ tô t€Jå&Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Y€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
zE ViT-Huge model (ViT-H/16) w/ no class token, avg pool @ 448x448
    r(  rÇ  rõ  Fr   r7  Úvit_huge_patch16_gap_448rØ  )r=  rî  rï  s       rS   r=  r=  «  r;  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )zH ViT-Giant (little-gg) model (ViT-g/16) w/ no class token, avg pool
    r(  r  r!  r   Fr   ©r.  r5  rJ   rB   rY   r6  r1  r<  Úvit_giant_patch16_gap_224rØ  )r@  rî  rï  s       rS   r@  r@  ¶  sn   € õ Ø ¨R¸2ÈØ u°eð=ñ =ô =€Jõ 'Ø#ðZð ZØ0:ðZÝ>BÀ:Ð>XÐ>XÐQWÐ>XÐ>XðZð Z€Eà€LrU   c           
      óŠ   — t          ddddt          t          d¬¦  «        ¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )NrÁ  r4  r  Trâ   ©rá   ©r5  rJ   rB   r:  rI   Úvit_xsmall_patch16_clip_224rØ  )rD  ©r_  r   r   ré  rï  s       rS   rD  rD  Â  ón   € õ  ¨2¸ÀTÕV]Õ^gÐmqÐVrÑVrÔVrÐsÑsÔs€JÝ&Ø%ð\ð \Ø2<ð\Ý@DÀZÐ@ZÐ@ZÐSYÐ@ZÐ@Zð\ð \€Eà€LrU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nrõ  rÅ  r+  r  Trâ   rB  ©r.  r5  rJ   rB   r:  rI   Úvit_medium_patch32_clip_224rØ  )rI  rE  rï  s       rS   rI  rI  Ë  sz   € õ Ø ¨B¸!ÀdÕW^Õ_hÐnrÐWsÑWsÔWsðuñ uô u€Jå&Ø%ð\ð \Ø2<ð\Ý@DÀZÐ@ZÐ@ZÐSYÐ@ZÐ@Zð\ð \€Eà€LrU   c           
      óŠ   — t          ddddt          t          d¬¦  «        ¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )NrÅ  r+  r  Trâ   rB  rC  Úvit_medium_patch16_clip_224rØ  )rK  rE  rï  s       rS   rK  rK  Õ  rF  rU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nrõ  rÊ  r+  r4  Trâ   rB  rH  Úvit_betwixt_patch32_clip_224rØ  )rM  rE  rï  s       rS   rM  rM  Þ  sz   € õ Ø ¨B¸"ÀtÕX_Õ`iÐosÐXtÑXtÔXtðvñ vô v€Jå&Ø&ð]ð ]Ø3=ð]ÝAEÀjÐA[ÐA[ÐTZÐA[ÐA[ð]ð ]€Eà€LrU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )ú) ViT-B/32 CLIP image tower @ 224x224
    rõ  r*  r+  Trâ   rB  rH  Úvit_base_patch32_clip_224rØ  )rP  rE  rï  s       rS   rP  rP  è  óz   € õ Ø ¨B¸"ÀtÕX_Õ`iÐosÐXtÑXtÔXtðvñ vô v€Jå&Ø#ðZð ZØ0:ðZÝ>BÀ:Ð>XÐ>XÐQWÐ>XÐ>XðZð Z€Eà€LrU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )z) ViT-B/32 CLIP image tower @ 256x256
    rõ  r*  r+  Trâ   rB  rH  Úvit_base_patch32_clip_256rØ  )rS  rE  rï  s       rS   rS  rS  ó  rQ  rU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )z) ViT-B/32 CLIP image tower @ 384x384
    rõ  r*  r+  Trâ   rB  rH  Úvit_base_patch32_clip_384rØ  )rU  rE  rï  s       rS   rU  rU  þ  rQ  rU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )z) ViT-B/32 CLIP image tower @ 448x448
    rõ  r*  r+  Trâ   rB  rH  Úvit_base_patch32_clip_448rØ  )rW  rE  rï  s       rS   rW  rW  	  rQ  rU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )z ViT-B/16 CLIP image tower
    r(  r*  r+  Trâ   rB  rH  Úvit_base_patch16_clip_224rØ  )rY  rE  rï  s       rS   rY  rY    rQ  rU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )z) ViT-B/16 CLIP image tower @ 384x384
    r(  r*  r+  Trâ   rB  rH  Úvit_base_patch16_clip_384rØ  )r[  rE  rï  s       rS   r[  r[    rQ  rU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )z5 ViT-Base (ViT-B/16+) CLIP image tower @ 240x240
    r(  rÍ  r+  r  Trâ   rB  rH  Úvit_base_patch16_plus_clip_240rØ  )r]  rE  rï  s       rS   r]  r]  *  sz   € õ Ø ¨B¸"ÀtÕX_Õ`iÐosÐXtÑXtÔXtðvñ vô v€Jå&Ø(ð_ð _Ø5?ð_ÝCGÈ
ÐC]ÐC]ÐV\ÐC]ÐC]ð_ð _€Eà€LrU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )z1 ViT-Large model (ViT-L/14) CLIP image tower
    r  rÆ  r  r(  Trâ   rB  rH  Úvit_large_patch14_clip_224rØ  )r_  rE  rï  s       rS   r_  r_  5  óz   € õ Ø ¨R¸2ÈÕY`ÕajÐptÐYuÑYuÔYuðwñ wô w€Jå&Ø$ð[ð [Ø1;ð[Ý?CÀJÐ?YÐ?YÐRXÐ?YÐ?Yð[ð [€Eà€LrU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )z; ViT-Large model (ViT-L/14) CLIP image tower @ 336x336
    r  rÆ  r  r(  Trâ   rB  rH  Úvit_large_patch14_clip_336rØ  )rb  rE  rï  s       rS   rb  rb  @  r`  rU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )z1 ViT-Huge model (ViT-H/14) CLIP image tower.
    r  rÇ  rõ  r(  Trâ   rB  rH  Úvit_huge_patch14_clip_224rØ  )rd  rE  rï  s       rS   rd  rd  K  óz   € õ Ø ¨R¸2ÈÕY`ÕajÐptÐYuÑYuÔYuðwñ wô w€Jå&Ø#ðZð ZØ0:ðZÝ>BÀ:Ð>XÐ>XÐQWÐ>XÐ>XðZð Z€Eà€LrU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )z: ViT-Huge model (ViT-H/14) CLIP image tower @ 336x336
    r  rÇ  rõ  r(  Trâ   rB  rH  Úvit_huge_patch14_clip_336rØ  )rg  rE  rï  s       rS   rg  rg  V  re  rU   c                 óŒ   — t          dddddt          t          d¬¦  «        ¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )z: ViT-Huge model (ViT-H/14) CLIP image tower @ 378x378
    r  rÇ  rõ  r(  Trâ   rB  rH  Úvit_huge_patch14_clip_378rØ  )ri  rE  rï  s       rS   ri  ri  a  re  rU   c                 óŽ   — t          ddddddt          t          d¬¦  «        ¬	¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )zŸ ViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    Pretrained weights from CLIP image tower.
    r  r  r   r!  r(  Trâ   rB  ©r.  r5  rY   rJ   rB   r:  rI   Úvit_giant_patch14_clip_224rØ  )rl  rE  rï  s       rS   rl  rl  l  sz   € õ
 Ø °¸bÈBÐY]Ý�9¨$Ð/Ñ/Ô/ðñ ô €Jõ 'Ø$ð[ð [Ø1;ð[Ý?CÀJÐ?YÐ?YÐRXÐ?YÐ?Yð[ð [€Eà€LrU   c                 óŽ   — t          ddddddt          t          d¬¦  «        ¬	¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )ú“ ViT-bigG model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    Pretrained weights from CLIP image tower.
    r  r%  r&  r'  r(  Trâ   rB  rk  Úvit_gigantic_patch14_clip_224rØ  )ro  rE  rï  s       rS   ro  ro  z  óz   € õ
 Ø °¸bÈBÐY]Ý�9¨$Ð/Ñ/Ô/ðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c                 óŽ   — t          ddddddt          t          d¬¦  «        ¬	¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )rn  r  r%  r&  r'  r(  Trâ   rB  rk  Úvit_gigantic_patch14_clip_378rØ  )rr  rE  rï  s       rS   rr  rr  ˆ  rp  rU   c                 óŽ   — t          dddddt          t          d¬¦  «        d¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )rO  rõ  r*  r+  Trâ   rB  Ú
quick_gelu©r.  r5  rJ   rB   r:  rI   r^   Ú#vit_base_patch32_clip_quickgelu_224rØ  )rv  rE  rï  s       rS   rv  rv  –  óy   € õ Ø ¨B¸"ÀtÝ�9¨$Ð/Ñ/Ô/¸<ðñ ô €Jõ 'Ø-ðdð dØ:DðdÝHLÈZÐHbÐHbÐ[aÐHbÐHbðdð d€Eà€LrU   c                 óŽ   — t          dddddt          t          d¬¦  «        d¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )z0 ViT-B/16 CLIP image tower w/ QuickGELU act
    r(  r*  r+  Trâ   rB  rt  ru  Ú#vit_base_patch16_clip_quickgelu_224rØ  )ry  rE  rï  s       rS   ry  ry  £  rw  rU   c                 óŽ   — t          dddddt          t          d¬¦  «        d¬	¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )zB ViT-Large model (ViT-L/14) CLIP image tower w/ QuickGELU act
    r  rÆ  r  r(  Trâ   rB  rt  ru  Ú$vit_large_patch14_clip_quickgelu_224rØ  )r{  rE  rï  s       rS   r{  r{  °  óy   € õ Ø ¨R¸2ÈÝ�9¨$Ð/Ñ/Ô/¸<ðñ ô €Jõ 'Ø.ðeð eØ;EðeÝIMÈjÐIcÐIcÐ\bÐIcÐIcðeð e€Eà€LrU   c                 óŽ   — t          dddddt          t          d¬¦  «        d¬	¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )zL ViT-Large model (ViT-L/14) CLIP image tower @ 336x336 w/ QuickGELU act
    r  rÆ  r  r(  Trâ   rB  rt  ru  Ú$vit_large_patch14_clip_quickgelu_336rØ  )r~  rE  rï  s       rS   r~  r~  ½  r|  rU   c                 óŽ   — t          dddddt          t          d¬¦  «        d¬	¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )zB ViT-Huge model (ViT-H/14) CLIP image tower w/ QuickGELU act.
    r  rÇ  rõ  r(  Trâ   rB  rt  ru  Ú#vit_huge_patch14_clip_quickgelu_224rØ  )r€  rE  rï  s       rS   r€  r€  Ê  óy   € õ Ø ¨R¸2ÈÝ�9¨$Ð/Ñ/Ô/¸<ðñ ô €Jõ 'Ø-ðdð dØ:DðdÝHLÈZÐHbÐHbÐ[aÐHbÐHbðdð d€Eà€LrU   c                 óŽ   — t          dddddt          t          d¬¦  «        d¬	¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )zK ViT-Huge model (ViT-H/14) CLIP image tower @ 378x378 w/ QuickGELU act
    r  rÇ  rõ  r(  Trâ   rB  rt  ru  Ú#vit_huge_patch14_clip_quickgelu_378rØ  )rƒ  rE  rï  s       rS   rƒ  rƒ  ×  r�  rU   c                 ó�   — t          ddddddt          t          d¬¦  «        d	¬
¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )z0 ViT-bigG model (ViT-G/14) w/ QuickGELU act
    r  r%  r&  r'  r(  Trâ   rB  rt  )r.  r5  rY   rJ   rB   r:  rI   r^   Ú'vit_gigantic_patch14_clip_quickgelu_224rØ  )r…  rE  rï  s       rS   r…  r…  ä  s}   € õ Ø °¸bÈBÐY]Ý�9¨$Ð/Ñ/Ô/¸<ðñ ô €Jõ 'Ø1ðhð hØ>HðhÝLPÐQ[ÐLfÐLfÐ_eÐLfÐLfðhð h€Eà€LrU   c           	      ób   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
z ViT-Base (ViT-B/32+)
    rõ  rÍ  r+  r  râ   ©r.  r5  rJ   rB   r\   Úvit_base_patch32_plus_256rØ  )rˆ  rî  rï  s       rS   rˆ  rˆ  ó  ó_   € õ  ¨s¸"ÈÐX\Ð]Ñ]Ô]€JÝ&Ø#ðZð ZØ0:ðZÝ>BÀ:Ð>XÐ>XÐQWÐ>XÐ>XðZð Z€Eà€LrU   c           	      ób   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
z ViT-Base (ViT-B/16+)
    r(  rÍ  r+  r  râ   r‡  Úvit_base_patch16_plus_240rØ  )r‹  rî  rï  s       rS   r‹  r‹  ý  r‰  rU   c                 ót   — t          dddddddt          d¬¦	  «	        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )z/ ViT-Base (ViT-B/16) w/ residual post-norm
    r(  r*  r+  Frâ   r   )	r.  r5  rJ   rB   rC   r\   r6  rN  r1  Úvit_base_patch16_rpn_224rØ  )r�  )r_  r’   ré  rï  s       rS   r�  r�    st   € õ Ø ¨B¸"ÀuÐZ^Ø¥L¸eðEñ Eô E€Jõ 'Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Y€Eà€LrU   c           	      ób   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
a   ViT-Base w/ LayerScale + 36 x 1 (36 block serial) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow.
    r(  r¼  é$   rö  râ   r‡  Úvit_small_patch16_36x1_224rØ  )r�  rî  rï  s       rS   r�  r�    s_   € õ  ¨s¸"ÈÐW[Ð\Ñ\Ô\€JÝ&Ø$ð[ð [Ø1;ð[Ý?CÀJÐ?YÐ?YÐRXÐ?YÐ?Yð[ð [€Eà€LrU   c           	      ón   — t          dddddt          ¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
a   ViT-Small w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow.
    r(  r¼  é   rö  râ   ©r.  r5  rJ   rB   r\   rN  Úvit_small_patch16_18x2_224rØ  )r”  ©r_  r  ré  rï  s       rS   r”  r”    sk   € õ Ø ¨B¸!ÈÕXkðmñ mô m€Jå&Ø$ð[ð [Ø1;ð[Ý?CÀJÐ?YÐ?YÐRXÐ?YÐ?Yð[ð [€Eà€LrU   c           	      ón   — t          dddddt          ¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
zÉ ViT-Base w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    r(  r*  r’  r+  râ   r“  Úvit_base_patch16_18x2_224rØ  )r—  r•  rï  s       rS   r—  r—  ,  sk   € õ
 Ø ¨B¸"È$ÕYlðnñ nô n€Jå&Ø#ðZð ZØ0:ðZÝ>BÀ:Ð>XÐ>XÐQWÐ>XÐ>XðZð Z€Eà€LrU   c           	      ób   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
zG EVA-large model https://arxiv.org/abs/2211.07636 /via MAE MIM pretrainr  rÆ  r  r(  r   ©r.  r5  rJ   rB   r1  Úeva_large_patch14_196rØ  )rš  rî  rï  s       rS   rš  rš  8  s^   € õ  ¨t¸2ÈÐY^Ð_Ñ_Ô_€JÝ&ØðVð VØ,6ðVÝ:>¸zÐ:TÐ:TÈVÐ:TÐ:TðVð V€Eà€LrU   c           	      ó`   — t          ddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
zF EVA-large model https://arxiv.org/abs/2211.07636 via MAE MIM pretrainr  rÆ  r  r(  r   r™  Úeva_large_patch14_336rØ  )rœ  rî  rï  s       rS   rœ  rœ  A  sM   € õ  ¨t¸2ÈÐY^Ð_Ñ_Ô_€JÝ&ÐtÐtÈ:ÐtÕY]Ð^hÐYsÐYsÐlrÐYsÐYsÐtÐt€EØ€LrU   c           	      ó`   — t          ddddd¬¦  «        }t          d	d| it          |fi |¤Ž¤Ž}|S )
z FlexiViT-Small
    r(  r¼  r+  rö  T©r.  r5  rJ   rB   r8  Úflexivit_smallrØ  )rŸ  rî  rï  s       rS   rŸ  rŸ  I  sM   € õ  ¨s¸"ÈÐZ^Ð_Ñ_Ô_€JÝ&ÐmÐmÀJÐmÕRVÐWaÐRlÐRlÐekÐRlÐRlÐmÐm€EØ€LrU   c           	      ó`   — t          ddddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	z FlexiViT-Base
    r(  r*  r+  Trž  Úflexivit_baserØ  )r¡  rî  rï  s       rS   r¡  r¡  R  sM   € õ  ¨s¸"ÈÐ[_Ð`Ñ`Ô`€JÝ&ÐlÐlÀ:ÐlÕQUÐV`ÐQkÐQkÐdjÐQkÐQkÐlÐl€EØ€LrU   c           	      ó`   — t          ddddd¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )	z FlexiViT-Large
    r(  rÆ  r  Trž  Úflexivit_largerØ  )r£  rî  rï  s       rS   r£  r£  [  sM   € õ  ¨t¸2ÈÐ\`ÐaÑaÔa€JÝ&ÐmÐmÀJÐmÕRVÐWaÐRlÐRlÐekÐRlÐRlÐmÐm€EØ€LrU   c                 ó€   — t          ddddddt          t          dd¬¦
  «
        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
úH ViT-Large model (ViT-L/14) w/ parallel blocks and qk norm enabled.
    r(  r*  r+  TF©
r.  r5  rJ   rB   r:  r8  rI   rN  rC   rD   Úvit_base_patch16_xp_224rØ  )r§  ©r_  r    rž   ré  rï  s       rS   r§  r§  d  st   € õ Ø ¨B¸"ÀtÐ\`ÝÕ%9ÀEÐSWðñ ô €Jõ 'Ø!ðXð XØ.8ðXÝ<@ÀÐ<VÐ<VÈvÐ<VÐ<VðXð X€Eà€LrU   c                 ó€   — t          ddddddt          t          dd¬¦
  «
        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )r¥  r  rÆ  r  r(  TFr¦  Úvit_large_patch14_xp_224rØ  )rª  r¨  rï  s       rS   rª  rª  q  su   € õ Ø ¨R¸2ÈÐ]aÝÕ%9ÀEÐSWðñ ô €Jõ 'Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Y€Eà€LrU   c                 ó€   — t          ddddddt          t          dd¬¦
  «
        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )zG ViT-Huge model (ViT-H/14) w/ parallel blocks and qk norm enabled.
    r  rÇ  rõ  r(  TFr¦  Úvit_huge_patch14_xp_224rØ  )r¬  r¨  rï  s       rS   r¬  r¬  ~  st   € õ Ø ¨R¸2ÈÐ]aÝÕ%9ÀEÐSWðñ ô €Jõ 'Ø!ðXð XØ.8ðXÝ<@ÀÐ<VÐ<VÈvÐ<VÐ<VðXð X€Eà€LrU   c           	      ób   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
z ViT-S/14 for DINOv2
    r  r¼  r+  rö  râ   r‡  Úvit_small_patch14_dinov2rØ  )r®  rî  rï  s       rS   r®  r®  ‹  s_   € õ  ¨s¸"ÈÐW[Ð\Ñ\Ô\€JÝ&Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Y€Eà€LrU   c           	      ób   — t          ddddd¬¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )	z ViT-B/14 for DINOv2
    r  r*  r+  râ   r‡  Úvit_base_patch14_dinov2rØ  )r°  rî  rï  s       rS   r°  r°  •  s^   € õ  ¨s¸"ÈÐX\Ð]Ñ]Ô]€JÝ&Ø!ðXð XØ.8ðXÝ<@ÀÐ<VÐ<VÈvÐ<VÐ<VðXð X€Eà€LrU   c           	      ób   — t          ddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
z ViT-L/14 for DINOv2
    r  rÆ  r  r(  râ   r‡  Úvit_large_patch14_dinov2rØ  )r²  rî  rï  s       rS   r²  r²  Ÿ  s_   € õ  ¨t¸2ÈÐY]Ð^Ñ^Ô^€JÝ&Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Y€Eà€LrU   c           
      ó†   — t          ddddddt          t          j        ¬¦  «        }t	          	 d
d	| it          |fi |¤Ž¤Ž}|S )ú ViT-G/14 for DINOv2
    r  é   r!  r  râ   çhË¹WU@)r.  r5  rJ   rB   r\   rY   r_   r^   Úvit_giant_patch14_dinov2rØ  )r·  ©r_  r   rn   ÚSiLUré  rï  s       rS   r·  r·  ©  sp   € õ Ø ¨R¸2È4Ø­ÅÄðñ ô €Jõ 'Ø"ðYð YØ/9ðYÝ=AÀ*Ð=WÐ=WÐPVÐ=WÐ=WðYð Y€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )z( ViT-S/14 for DINOv2 w/ 4 registers
    r  r¼  r+  rö  râ   r  T©r.  r5  rJ   rB   r\   r9  r8  Úvit_small_patch14_reg4_dinov2rØ  )r¼  rî  rï  s       rS   r¼  r¼  º  sl   € õ Ø ¨B¸!ÈØ Tðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )z( ViT-B/14 for DINOv2 w/ 4 registers
    r  r*  r+  râ   r  Tr»  Úvit_base_patch14_reg4_dinov2rØ  )r¾  rî  rï  s       rS   r¾  r¾  Ç  sl   € õ Ø ¨B¸"È$Ø Tðñ ô €Jõ 'Ø&ð]ð ]Ø3=ð]ÝAEÀjÐA[ÐA[ÐTZÐA[ÐA[ð]ð ]€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )z( ViT-L/14 for DINOv2 w/ 4 registers
    r  rÆ  r  r(  râ   r  Tr»  Úvit_large_patch14_reg4_dinov2rØ  )rÀ  rî  rï  s       rS   rÀ  rÀ  Ô  sl   € õ Ø ¨R¸2È4Ø Tðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c                 óŠ   — t          ddddddt          t          j        dd¬	¦
  «
        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )r´  r  rµ  r!  r  râ   r¶  r  T©
r.  r5  rJ   rB   r\   rY   r_   r^   r9  r8  Úvit_giant_patch14_reg4_dinov2rØ  )rÃ  r¸  rï  s       rS   rÃ  rÃ  á  sv   € õ Ø ¨R¸2È4Ð[fÝ­"¬'¸aÐPTðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )zR ViT-B/14 for TIPSv2 (DINOv2-style w/ 1 register token, LayerScale init=1.0).
    r  r*  r+  r½  r0   Tr»  Úvit_base_patch14_reg1_tipsv2rØ  )rÅ  rî  rï  s       rS   rÅ  rÅ  ò  sl   € õ Ø ¨B¸"È#Ø Tðñ ô €Jõ 'Ø&ð]ð ]Ø3=ð]ÝAEÀjÐA[ÐA[ÐTZÐA[ÐA[ð]ð ]€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )zR ViT-L/14 for TIPSv2 (DINOv2-style w/ 1 register token, LayerScale init=1.0).
    r  rÆ  r  r(  r½  r0   Tr»  Úvit_large_patch14_reg1_tipsv2rØ  )rÇ  rî  rï  s       rS   rÇ  rÇ  ÿ  sl   € õ Ø ¨R¸2È3Ø Tðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c           
      óh   — t          dddddddd¬	¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )zW SoViT-400M/14 for TIPSv2 (DINOv2-style w/ 1 register token, LayerScale init=1.0).
    r  é€  é   r(  r½  g9Žã8Žã@r0   T)r.  r5  rJ   rB   r\   rY   r9  r8  Úvit_so400m_patch14_reg1_tipsv2rØ  )rË  rî  rï  s       rS   rË  rË    sn   € õ Ø ¨R¸2È3Ø¨!¸Dðñ ô €Jõ 'Ø(ð_ð _Ø5?ð_ÝCGÈ
ÐC]ÐC]ÐV\ÐC]ÐC]ð_ð _€Eà€LrU   c                 óŠ   — t          ddddddt          t          j        dd¬	¦
  «
        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )z^ ViT-G/14 for TIPSv2 (DINOv2-style w/ SwiGLU FFN, 1 register token, LayerScale init=1.0).
    r  rµ  r!  r  r½  r¶  r0   TrÂ  Úvit_giant_patch14_reg1_tipsv2rØ  )rÍ  r¸  rï  s       rS   rÍ  rÍ    su   € õ Ø ¨R¸2È3Ø­ÅÄØ Tðñ ô €Jõ
 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )Nrõ  r*  r+  Fr3  Ú	gelu_tanh©r.  r5  rJ   rB   r6  r1  r^   Úvit_base_patch32_siglip_256rØ  )rÑ  rî  rï  s       rS   rÑ  rÑ  )  sk   € åØ ¨B¸"È%Ð]bØðñ ô €Jõ 'Ø%ð\ð \Ø2<ð\Ý@DÀZÐ@ZÐ@ZÐSYÐ@ZÐ@Zð\ð \€Eà€LrU   c           	      ód   — t          dddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
Nr(  r*  r+  Fr3  ©r.  r5  rJ   rB   r6  r1  Úvit_base_patch16_siglip_224rØ  )rÔ  rî  rï  s       rS   rÔ  rÔ  4  óh   € åØ ¨B¸"È%Ð]bðñ ô €Jõ 'Ø%ð\ð \Ø2<ð\Ý@DÀZÐ@ZÐ@ZÐSYÐ@ZÐ@Zð\ð \€Eà€LrU   c           	      ód   — t          dddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
Nr(  r*  r+  Fr3  rÓ  Úvit_base_patch16_siglip_256rØ  )r×  rî  rï  s       rS   r×  r×  >  rÕ  rU   c           	      ód   — t          dddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
Nr(  r*  r+  Fr3  rÓ  Úvit_base_patch16_siglip_384rØ  )rÙ  rî  rï  s       rS   rÙ  rÙ  H  rÕ  rU   c           	      ód   — t          dddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
Nr(  r*  r+  Fr3  rÓ  Úvit_base_patch16_siglip_512rØ  )rÛ  rî  rï  s       rS   rÛ  rÛ  R  rÕ  rU   c           	      ód   — t          dddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
Nr(  rÆ  r  Fr3  rÓ  Úvit_large_patch16_siglip_256rØ  )rÝ  rî  rï  s       rS   rÝ  rÝ  \  óh   € åØ ¨R¸2È5Ð^cðñ ô €Jõ 'Ø&ð]ð ]Ø3=ð]ÝAEÀjÐA[ÐA[ÐTZÐA[ÐA[ð]ð ]€Eà€LrU   c           	      ód   — t          dddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
Nr(  rÆ  r  Fr3  rÓ  Úvit_large_patch16_siglip_384rØ  )rà  rî  rï  s       rS   rà  rà  f  rÞ  rU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )Nr(  rÆ  r  Fr3  rÏ  rÐ  Úvit_large_patch16_siglip_512rØ  )râ  rî  rï  s       rS   râ  râ  p  sk   € åØ ¨R¸2È5Ð^cØðñ ô €Jõ 'Ø&ð]ð ]Ø3=ð]ÝAEÀjÐA[ÐA[ÐTZÐA[ÐA[ð]ð ]€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr  rÉ  rÊ  r(  çˆ…ZÓ¼ã@Fr3  ©r.  r5  rJ   rB   rY   r6  r1  Úvit_so400m_patch14_siglip_224rØ  )ræ  rî  rï  s       rS   ræ  ræ  {  ók   € åØ ¨R¸2ÈÐ]bÐpuðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr  rÉ  rÊ  r(  rä  Fr3  rå  Úvit_so400m_patch14_siglip_378rØ  )ré  rî  rï  s       rS   ré  ré  …  sm   € õ Ø ¨R¸2ÈÐ]bÐpuðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr  rÉ  rÊ  r(  rä  Fr3  rå  Úvit_so400m_patch14_siglip_384rØ  )rë  rî  rï  s       rS   rë  rë  �  rç  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr(  rÉ  rÊ  rä  Fr3  rÏ  ©r.  r5  rJ   rB   rY   r6  r1  r^   Úvit_so400m_patch16_siglip_256rØ  )rî  rî  rï  s       rS   rî  rî  š  ón   € åØ ¨R¸2ÈÐ]bÐpuØðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c           
      óh   — t          dddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr(  rÉ  rÊ  rä  Fr3  rÏ  rí  Úvit_so400m_patch16_siglip_384rØ  )rñ  rî  rï  s       rS   rñ  rñ  ¥  rï  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr(  rÉ  rÊ  rä  Fr3  rÏ  rí  Úvit_so400m_patch16_siglip_512rØ  )ró  rî  rï  s       rS   ró  ró  °  rï  rU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )Nr(  rµ  r!  Fr3  rÏ  rÐ  Úvit_giantopt_patch16_siglip_256rØ  )rõ  rî  rï  s       rS   rõ  rõ  »  ók   € åØ ¨R¸2È5Ð^cØðñ ô €Jõ 'Ø)ð`ð `Ø6@ð`ÝDHÈÐD^ÐD^ÐW]ÐD^ÐD^ð`ð `€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )Nr(  rµ  r!  Fr3  rÏ  rÐ  Úvit_giantopt_patch16_siglip_384rØ  )rø  rî  rï  s       rS   rø  rø  Æ  rö  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )Nrõ  r*  r+  Fr   rÏ  ©r.  r5  rJ   rB   r6  r1  r<  r^   Úvit_base_patch32_siglip_gap_256rØ  )rû  rî  rï  s       rS   rû  rû  Ñ  sn   € åØ ¨B¸"È%Ð]bÐlqØðñ ô €Jõ 'Ø)ð`ð `Ø6@ð`ÝDHÈÐD^ÐD^ÐW]ÐD^ÐD^ð`ð `€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
ú^ A SigLIP variant of ViT with global average pooling (GAP) instead of attention pooling (MAP).r(  r*  r+  Fr   r7  Úvit_base_patch16_siglip_gap_224rØ  )rþ  rî  rï  s       rS   rþ  rþ  Ü  óm   € õ Ø ¨B¸"È%Ð]bÐlqðñ ô €Jõ 'Ø)ð`ð `Ø6@ð`ÝDHÈÐD^ÐD^ÐW]ÐD^ÐD^ð`ð `€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
rý  r(  r*  r+  Fr   r7  Úvit_base_patch16_siglip_gap_256rØ  )r  rî  rï  s       rS   r  r  ç  rÿ  rU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
rý  r(  r*  r+  Fr   r7  Úvit_base_patch16_siglip_gap_384rØ  )r  rî  rï  s       rS   r  r  ò  rÿ  rU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
rý  r(  r*  r+  Fr   r7  Úvit_base_patch16_siglip_gap_512rØ  )r  rî  rï  s       rS   r  r  ý  rÿ  rU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
rý  r(  rÆ  r  Fr   r7  Ú vit_large_patch16_siglip_gap_256rØ  )r  rî  rï  s       rS   r  r    óm   € õ Ø ¨R¸2È5Ð^cÐmrðñ ô €Jõ 'Ø*ðað aØ7AðaÝEIÈ*ÐE_ÐE_ÐX^ÐE_ÐE_ðað a€Eà€LrU   c           	      óf   — t          ddddddd¬¦  «        }t          	 d	d| it          |fi |¤Ž¤Ž}|S )
rý  r(  rÆ  r  Fr   r7  Ú vit_large_patch16_siglip_gap_384rØ  )r
  rî  rï  s       rS   r
  r
    r  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )Nr(  rÆ  r  Fr   rÏ  rú  Ú vit_large_patch16_siglip_gap_512rØ  )r  rî  rï  s       rS   r  r    sl   € åØ ¨R¸2È5Ø 5°Kðñ ô €Jõ 'Ø*ðað aØ7AðaÝEIÈ*ÐE_ÐE_ÐX^ÐE_ÐE_ðað a€Eà€LrU   c           
      óh   — t          dddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )rý  r  rÉ  rÊ  r(  rä  Fr   r?  Ú!vit_so400m_patch14_siglip_gap_224rØ  )r  rî  rï  s       rS   r  r  )  ón   € õ Ø ¨R¸2ÈØ u°eðñ ô €Jõ 'Ø+ðbð bØ8BðbÝFJÈ:ÐF`ÐF`ÐY_ÐF`ÐF`ðbð b€Eà€LrU   c           
      óh   — t          dddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )rý  r  rÉ  rÊ  r(  rä  Fr   r?  Ú!vit_so400m_patch14_siglip_gap_378rØ  )r  rî  rï  s       rS   r  r  5  r  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )rý  r  rÉ  rÊ  r(  rä  Fr   r?  Ú!vit_so400m_patch14_siglip_gap_384rØ  )r  rî  rï  s       rS   r  r  A  r  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )rý  r  rÉ  rÊ  r(  rä  Fr   r?  Ú!vit_so400m_patch14_siglip_gap_448rØ  )r  rî  rï  s       rS   r  r  M  r  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )rý  r  rÉ  rÊ  r(  rä  Fr   r?  Ú!vit_so400m_patch14_siglip_gap_896rØ  )r  rî  rï  s       rS   r  r  Y  r  rU   c                 ój   — t          ddddddddd¬¦	  «	        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )rý  r(  rÉ  rÊ  rä  Fr   rÏ  ©	r.  r5  rJ   rB   rY   r6  r1  r<  r^   Ú!vit_so400m_patch16_siglip_gap_256rØ  )r  rî  rï  s       rS   r  r  e  sp   € õ Ø ¨R¸2ÈØ u°eÀ{ðñ ô €Jõ 'Ø+ðbð bØ8BðbÝFJÈ:ÐF`ÐF`ÐY_ÐF`ÐF`ðbð b€Eà€LrU   c                 ój   — t          ddddddddd¬¦	  «	        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr(  rÉ  rÊ  rä  Fr   rÏ  r  Ú!vit_so400m_patch16_siglip_gap_384rØ  )r  rî  rï  s       rS   r  r  q  óo   € åØ ¨R¸2ÈÐ]bØ 5°Kðñ ô €Jõ 'Ø+ðbð bØ8BðbÝFJÈ:ÐF`ÐF`ÐY_ÐF`ÐF`ðbð b€Eà€LrU   c                 ój   — t          ddddddddd¬¦	  «	        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr(  rÉ  rÊ  rä  Fr   rÏ  r  Ú!vit_so400m_patch16_siglip_gap_512rØ  )r  rî  rï  s       rS   r  r  |  r  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )Nr(  rµ  r!  Fr   rÏ  rú  Ú#vit_giantopt_patch16_siglip_gap_256rØ  )r!  rî  rï  s       rS   r!  r!  ‡  ól   € åØ ¨R¸2È5Ø 5°Kðñ ô €Jõ 'Ø-ðdð dØ:DðdÝHLÈZÐHbÐHbÐ[aÐHbÐHbðdð d€Eà€LrU   c           
      óh   — t          dddddddd¬¦  «        }t          	 d
d	| it          |fi |¤Ž¤Ž}|S )Nr(  rµ  r!  Fr   rÏ  rú  Ú#vit_giantopt_patch16_siglip_gap_384rØ  )r$  rî  rï  s       rS   r$  r$  ’  r"  rU   c                 ól   — t          ddddddddd	d
¬¦
  «
        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÁ  r  r  râ   é   FTr0   r   ©
r.  r5  rJ   rB   r\   rY   r6  r8  r9  r1  Úvit_wee_patch16_reg1_gap_256rØ  )r(  rî  rï  s       rS   r(  r(  �  sq   € åØ ¨B¸!ÈÐYZØ¨$¸1È%ðñ ô €Jõ 'Ø&ð]ð ]Ø3=ð]ÝAEÀjÐA[ÐA[ÐTZÐA[ÐA[ð]ð ]€Eà€LrU   c                 ón   — t          ddddddddd	d
d¬¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÁ  r  r  râ   r&  FTr0   r   r>   ©r.  r5  rJ   rB   r\   rY   r6  r8  r9  r1  r@   Úvit_dwee_patch16_reg1_gap_256rØ  )r+  rî  rï  s       rS   r+  r+  ¨  st   € åØ ¨B¸!ÈÐYZØ¨$¸1È%Ð\bðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c                 óx   — t          dddddddddd	t          ¬
¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÁ  r  râ   r&  FTr0   r   ©r.  r5  rJ   rB   r\   rY   r6  r8  r9  r1  rN  Úvit_pwee_patch16_reg1_gap_256rØ  )r.  )r_  rž   ré  rï  s       rS   r.  r.  ³  st   € åØ ¨B¸!ÈÐYZØ¨$¸1È%ÕZnðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c                 óx   — t          dddddddddd	t          ¬
¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÁ  r  râ   r&  FTr0   r   r-  Úvit_dpwee_patch16_reg1_gap_256rØ  )r0  )r_  rÞ   ré  rï  s       rS   r0  r0  ¾  st   € åØ ¨B¸!ÈÐYZØ¨$¸1È%ÕZrðñ ô €Jõ 'Ø(ð_ð _Ø5?ð_ÝCGÈ
ÐC]ÐC]ÐV\ÐC]ÐC]ð_ð _€Eà€LrU   c                 ól   — t          ddddddddd	d
¬¦
  «
        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  é@  r  r&  râ   çffffff@FTr0   r   r'  Úvit_little_patch16_reg1_gap_256rØ  )r4  rî  rï  s       rS   r4  r4  É  óq   € åØ ¨B¸!ÈÐY\Ø¨$¸1È%ðñ ô €Jõ 'Ø)ð`ð `Ø6@ð`ÝDHÈÐD^ÐD^ÐW]ÐD^ÐD^ð`ð `€Eà€LrU   c                 ón   — t          ddddddddd	d
d¬¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  r2  r  r&  râ   r3  FTr0   r   r>   r*  Ú vit_dlittle_patch16_reg1_gap_256rØ  )r7  rî  rï  s       rS   r7  r7  Ô  st   € åØ ¨B¸!ÈÐY\Ø¨$¸1È%Ð\bðñ ô €Jõ 'Ø*ðað aØ7AðaÝEIÈ*ÐE_ÐE_ÐX^ÐE_ÐE_ðað a€Eà€LrU   c                 ól   — t          ddddddddd	d
¬¦
  «
        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  r2  r  r&  râ   r3  FTr  r   r'  Úvit_little_patch16_reg4_gap_256rØ  )r9  rî  rï  s       rS   r9  r9  ß  r5  rU   c                 ój   — t          ddddddddd	¬
¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÅ  r+  r  râ   FTr0   r   ©	r.  r5  rJ   rB   r\   r6  r8  r9  r1  Úvit_medium_patch16_reg1_gap_256rØ  )r<  rî  rï  s       rS   r<  r<  ê  ón   € åØ ¨B¸!ÈØ¨$¸1È%ðñ ô €Jõ 'Ø)ð`ð `Ø6@ð`ÝDHÈÐD^ÐD^ÐW]ÐD^ÐD^ð`ð `€Eà€LrU   c                 ój   — t          ddddddddd	¬
¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÅ  r+  r  râ   FTr  r   r;  Úvit_medium_patch16_reg4_gap_256rØ  )r?  rî  rï  s       rS   r?  r?  õ  r=  rU   c                 ój   — t          ddddddddd	¬
¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÅ  é   r  râ   FTr  r   r;  Ú vit_mediumd_patch16_reg4_gap_256rØ  )rB  rî  rï  s       rS   rB  rB     ón   € åØ ¨B¸!ÈØ¨$¸1È%ðñ ô €Jõ 'Ø*ðað aØ7AðaÝEIÈ*ÐE_ÐE_ÐX^ÐE_ÐE_ðað a€Eà€LrU   c                 ój   — t          ddddddddd	¬
¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÅ  rA  r  râ   FTr  r   r;  Ú vit_mediumd_patch16_reg4_gap_384rØ  )rE  rî  rï  s       rS   rE  rE    rC  rU   c                 ój   — t          ddddddddd	¬
¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÊ  r+  r4  râ   FTr0   r   r;  Ú vit_betwixt_patch16_reg1_gap_256rØ  )rG  rî  rï  s       rS   rG  rG    ón   € åØ ¨B¸"È$Ø¨$¸1È%ðñ ô €Jõ 'Ø*ðað aØ7AðaÝEIÈ*ÐE_ÐE_ÐX^ÐE_ÐE_ðað a€Eà€LrU   c                 ój   — t          ddddddddd	¬
¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÊ  r+  r4  râ   FTr  r   r;  Ú vit_betwixt_patch16_reg4_gap_256rØ  )rJ  rî  rï  s       rS   rJ  rJ  !  rH  rU   c                 ój   — t          ddddddddd	¬
¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )Nr(  rÊ  r+  r4  râ   FTr  r   r;  Ú vit_betwixt_patch16_reg4_gap_384rØ  )rL  rî  rï  s       rS   rL  rL  ,  rH  rU   c           
      óh   — t          dddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr(  r*  r+  FTr   r  )r.  r5  rJ   rB   r6  r8  r1  r9  Úvit_base_patch16_reg4_gap_256rØ  )rN  rî  rï  s       rS   rN  rN  7  sl   € åØ ¨B¸"È%Ø¨¸1ðñ ô €Jõ 'Ø'ð^ð ^Ø4>ð^ÝBFÀzÐB\ÐB\ÐU[ÐB\ÐB\ð^ð ^€Eà€LrU   c           
      óh   — t          dddddddd¬	¦  «        }t          	 dd| it          |fi |¤Ž¤Ž}|S )úF SO150M (shape optimized, but diff than paper def, optimized for GPU) r(  rÍ  r’  r  çú~j¼t“@Fr  r3  )r.  r5  rJ   rB   rY   r6  r9  r1  Úvit_so150m_patch16_reg4_map_256rØ  )rR  rî  rï  s       rS   rR  rR  B  sn   € õ Ø ¨B¸"ÈØ a°Uðñ ô €Jõ 'Ø)ð`ð `Ø6@ð`ÝDHÈÐD^ÐD^ÐW]ÐD^ÐD^ð`ð `€Eà€LrU   c                 ój   — t          ddddddddd¬	¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )rP  r(  rÍ  r’  r  rQ  Fr  r   ©	r.  r5  rJ   rB   rY   r6  r9  r1  r<  Úvit_so150m_patch16_reg4_gap_256rØ  )rU  rî  rï  s       rS   rU  rU  N  óp   € õ Ø ¨B¸"ÈØ a°UÀEðñ ô €Jõ 'Ø)ð`ð `Ø6@ð`ÝDHÈÐD^ÐD^ÐW]ÐD^ÐD^ð`ð `€Eà€LrU   c                 ój   — t          ddddddddd¬	¦	  «	        }t          	 dd| it          |fi |¤Ž¤Ž}|S )rP  r(  rÍ  r’  r  rQ  Fr  r   rT  Úvit_so150m_patch16_reg4_gap_384rØ  )rX  rî  rï  s       rS   rX  rX  Z  rV  rU   c                 ól   — t          dddddddddd	¬
¦
  «
        }t          	 dd| it          |fi |¤Ž¤Ž}|S )úI SO150M v2 (shape optimized, but diff than paper def, optimized for GPU) r(  é@  é   é   çÅNìÄNì@râ   Fr0   r   ©
r.  r5  rJ   rB   rY   r\   rC   r6  r9  r1  Ú vit_so150m2_patch16_reg1_gap_256rØ  )r`  rî  rï  s       rS   r`  r`  f  ós   € õ Ø ¨B¸"ÈÐ[_Ø E°aÀUðñ ô €Jõ 'Ø*ðað aØ7AðaÝEIÈ*ÐE_ÐE_ÐX^ÐE_ÐE_ðað a€Eà€LrU   c                 ól   — t          dddddddddd	¬
¦
  «
        }t          	 dd| it          |fi |¤Ž¤Ž}|S )rZ  r(  r[  r\  r]  r^  râ   Fr0   r   r_  Ú vit_so150m2_patch16_reg1_gap_384rØ  )rc  rî  rï  s       rS   rc  rc  r  ra  rU   c                 ól   — t          dddddddddd	¬
¦
  «
        }t          	 dd| it          |fi |¤Ž¤Ž}|S )rZ  r(  r[  r\  r]  r^  râ   Fr0   r   r_  Ú vit_so150m2_patch16_reg1_gap_448rØ  )re  rî  rï  s       rS   re  re  ~  ra  rU   c           	      óf   — t          ddddddd¬¦  «        }t          	 dd
| it          |fi |¤Ž¤Ž}|S )Nr  rÆ  r  r(  ró   FT)r.  r5  rJ   rB   r\   r;  r&  Úvit_intern300m_patch14_448rØ  )rg  rî  rï  s       rS   rg  rg  Š  sj   € åØ ¨R¸2Ø E¸Dðñ ô €Jõ 'Ø$ð[ð [Ø1;ð[Ý?CÀJÐ?YÐ?YÐRXÐ?YÐ?Yð[ð [€Eà€LrU   c                 óÎ   — t          dddddddddddt          t          d	¬
¦  «        t          t          d	¬
¦  «        t          ¬¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )ú ViT Large AIM-v2 model
    r  rÆ  r  r  Fç      @r   Úsilurâ   rB  ©r.  r5  rJ   rB   r6  r<  rY   r1  rC   rF   r^   rI   rM  r_   Úaimv2_large_patch14_224rØ  )rm  ©r_  r   r    r   ré  rï  s       rS   rm  rm  •  ó˜   € õ Ø ¨R¸1È%ÐY^Ø E°EÀUÐV\Ý�7¨Ð-Ñ-Ô-ÅÍÐUYÐ@ZÑ@ZÔ@ZÕflðñ ô €Jõ
 'Ø!ðXð XØ.8ðXÝ<@ÀÐ<VÐ<VÈvÐ<VÐ<VðXð X€Eà€LrU   c                 óÎ   — t          dddddddddddt          t          d	¬
¦  «        t          t          d	¬
¦  «        t          ¬¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )ú ViT Huge AIM-v2 model
    r  rµ  r  r+  FçòAÏfU@r   rk  râ   rB  rl  Úaimv2_huge_patch14_224rØ  )rs  rn  rï  s       rS   rs  rs  £  s˜   € õ
 Ø ¨R¸2È5ÐZ_Ø e°eÀuÐX^Ý�7¨Ð-Ñ-Ô-ÅÍÐUYÐ@ZÑ@ZÔ@ZÕflðñ ô €Jõ
 'Ø ðWð WØ-7ðWÝ;?À
Ð;UÐ;UÈfÐ;UÐ;UðWð W€Eà€LrU   c                 óÎ   — t          dddddddddddt          t          d	¬
¦  «        t          t          d	¬
¦  «        t          ¬¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )ú ViT 1B AIM-v2 model
    r  é   r  r(  Frj  r   rk  râ   rB  rl  Úaimv2_1b_patch14_224rØ  )rw  rn  rï  s       rS   rw  rw  ²  ó˜   € õ Ø ¨R¸2È5ÐZ_Ø E°EÀUÐV\Ý�7¨Ð-Ñ-Ô-ÅÍÐUYÐ@ZÑ@ZÔ@ZÕflðñ ô €Jõ
 'ØðUð UØ+5ðUÝ9=¸jÐ9SÐ9SÈFÐ9SÐ9SðUð U€Eà€LrU   c                 óÎ   — t          dddddddddddt          t          d¬	¦  «        t          t          d¬	¦  «        t          ¬
¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )ú ViT 3B AIM-v2 model
    r  é   r  Frr  r   rk  râ   rB  rl  Úaimv2_3b_patch14_224rØ  )r|  rn  rï  s       rS   r|  r|  À  ó˜   € õ Ø ¨R¸2È5ÐZ_Ø e°eÀuÐX^Ý�7¨Ð-Ñ-Ô-ÅÍÐUYÐ@ZÑ@ZÔ@ZÕflðñ ô €Jõ
 'ØðUð UØ+5ðUÝ9=¸jÐ9SÐ9SÈFÐ9SÐ9SðUð U€Eà€LrU   c                 óÎ   — t          dddddddddddt          t          d	¬
¦  «        t          t          d	¬
¦  «        t          ¬¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )ri  r  rÆ  r  r  Frj  r   rk  râ   rB  rl  Úaimv2_large_patch14_336rØ  )r  rn  rï  s       rS   r  r  Î  ro  rU   c                 óÎ   — t          dddddddddddt          t          d	¬
¦  «        t          t          d	¬
¦  «        t          ¬¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )rq  r  rµ  r  r+  Frr  r   rk  râ   rB  rl  Úaimv2_huge_patch14_336rØ  )r�  rn  rï  s       rS   r�  r�  Ü  ó˜   € õ Ø ¨R¸2È5ÐZ_Ø e°eÀuÐX^Ý�7¨Ð-Ñ-Ô-ÅÍÐUYÐ@ZÑ@ZÔ@ZÕflðñ ô €Jõ
 'Ø ðWð WØ-7ðWÝ;?À
Ð;UÐ;UÈfÐ;UÐ;UðWð W€Eà€LrU   c                 óÎ   — t          dddddddddddt          t          d	¬
¦  «        t          t          d	¬
¦  «        t          ¬¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )ru  r  rv  r  r(  Frj  r   rk  râ   rB  rl  Úaimv2_1b_patch14_336rØ  )r„  rn  rï  s       rS   r„  r„  ê  rx  rU   c                 óÎ   — t          dddddddddddt          t          d¬	¦  «        t          t          d¬	¦  «        t          ¬
¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )rz  r  r{  r  Frr  r   rk  râ   rB  rl  Úaimv2_3b_patch14_336rØ  )r†  rn  rï  s       rS   r†  r†  ø  r}  rU   c                 óÎ   — t          dddddddddddt          t          d	¬
¦  «        t          t          d	¬
¦  «        t          ¬¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )ri  r  rÆ  r  r  Frj  r   rk  râ   rB  rl  Úaimv2_large_patch14_448rØ  )rˆ  rn  rï  s       rS   rˆ  rˆ    ro  rU   c                 óÎ   — t          dddddddddddt          t          d	¬
¦  «        t          t          d	¬
¦  «        t          ¬¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )rq  r  rµ  r  r+  Frr  r   rk  râ   rB  rl  Úaimv2_huge_patch14_448rØ  )rŠ  rn  rï  s       rS   rŠ  rŠ    r‚  rU   c                 óÎ   — t          dddddddddddt          t          d	¬
¦  «        t          t          d	¬
¦  «        t          ¬¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )ru  r  rv  r  r(  Frj  r   rk  râ   rB  rl  Úaimv2_1b_patch14_448rØ  )rŒ  rn  rï  s       rS   rŒ  rŒ  "  rx  rU   c                 óÎ   — t          dddddddddddt          t          d¬	¦  «        t          t          d¬	¦  «        t          ¬
¦  «        }t	          	 dd| it          |fi |¤Ž¤Ž}|S )rz  r  r{  r  Frr  r   rk  râ   rB  rl  Úaimv2_3b_patch14_448rØ  )rŽ  rn  rï  s       rS   rŽ  rŽ  0  r}  rU   c           	      ób   — t          dddddd¬¦  «        }t          d
d	| it          |fi |¤Ž¤Ž}|S )ú ViT Test
    r(  é@   rö  rÂ   r¥   T)r.  r5  rJ   rB   rY   r&  Útest_vitrØ  )r’  rî  rï  s       rS   r’  r’  >  sO   € õ  ¨r¸ÀaÐSTÐgkÐlÑlÔl€JÝ&ÐgÐg¸jÐgÍDÐQ[ÐLfÐLfÐ_eÐLfÐLfÐgÐg€EØ€LrU   c                 ój   — t          ddddddddd	d
¬¦
  «
        }t          dd| it          |fi |¤Ž¤Ž}|S )r�  r(  r‘  r  rÂ   r¥   Fr0   r   râ   T)
r.  r5  rJ   rB   rY   r6  r9  r1  r\   r&  Ú	test_vit2rØ  )r”  rî  rï  s       rS   r”  r”  G  sc   € õ Ø ¨1¸ÀQØ a°UÈÐ_cðeñ eô e€Jõ 'ÐhÐh¸zÐhÍTÐR\ÐMgÐMgÐ`fÐMgÐMgÐhÐh€EØ€LrU   c                 ój   — t          ddddddddd	d
¬¦
  «
        }t          dd| it          |fi |¤Ž¤Ž}|S )r�  r(  é`   é	   r¥   rÂ   Fr0   r3  Trâ   )
r.  r5  rJ   rB   rY   r6  r9  r1  r=  r\   Ú	test_vit3rØ  )r˜  rî  rï  s       rS   r˜  r˜  R  sd   € õ Ø ¨1¸ÀQØ a°UÐPTÐbfðhñ hô h€Jõ 'ÐhÐh¸zÐhÍTÐR\ÐMgÐMgÐ`fÐMgÐMgÐhÐh€EØ€LrU   c                 ól   — t          dddddddddd	d
¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )r�  r(  r–  r—  r¥   Fr0   r   râ   TÚrmsnorm)r.  r5  rJ   rB   rY   r6  r9  r1  r\   r&  rI   Ú	test_vit4rØ  )r›  rî  rï  s       rS   r›  r›  ]  sc   € õ Ø ¨1¸ÀQØ a°UÈÐ_cØðñ ô €Jõ
 'ÐhÐh¸zÐhÍTÐR\ÐMgÐMgÐ`fÐMgÐMgÐhÐh€EØ€LrU   c                 ó’   — t          dddddddddt          t          d¬¦  «        ¬	¦
  «
        }t          dd| it          |fi |¤Ž¤Ž}|S )zu BEiT3 Base model (ViT-Base size) with patch size 16x16.
    Remapped to VisionTransformer with scale_norm=True.
    r(  r*  r+  r  Tr   râ   rB  ©
r.  r5  rJ   rB   rY   rZ   r[   r6  r1  rI   Úbeit3_base_patch16_224rØ  )rž  rE  rï  s       rS   rž  rž  j  so   € õ
 Ø ¨B¸"ÈØ¨T¸tÐQVÝ�9¨$Ð/Ñ/Ô/ðñ ô €Jõ
 'ÐuÐuÈJÐuÕZ^Ð_iÐZtÐZtÐmsÐZtÐZtÐuÐu€EØ€LrU   c                 ó’   — t          dddddddddt          t          d¬¦  «        ¬	¦
  «
        }t          dd| it          |fi |¤Ž¤Ž}|S )zw BEiT3 Large model (ViT-Large size) with patch size 16x16.
    Remapped to VisionTransformer with scale_norm=True.
    r(  rÆ  r  r  Tr   râ   rB  r�  Úbeit3_large_patch16_224rØ  )r   rE  rï  s       rS   r   r   x  so   € õ
 Ø ¨R¸2ÈØ¨T¸tÐQVÝ�9¨$Ð/Ñ/Ô/ðñ ô €Jõ
 'ÐvÐvÈZÐvÕ[_Ð`jÐ[uÐ[uÐntÐ[uÐ[uÐvÐv€EØ€LrU   c                 ó’   — t          dddddddddt          t          d¬	¦  «        ¬
¦
  «
        }t          dd| it          |fi |¤Ž¤Ž}|S )zf BEiT3 Giant model with patch size 14x14.
    Remapped to VisionTransformer with scale_norm=True.
    r  r  r!  r(  ç†8ÖÅmt@Tr   râ   rB  r�  Úbeit3_giant_patch14_224rØ  )r£  rE  rï  s       rS   r£  r£  †  so   € õ
 Ø ¨R¸2ÈØ¨T¸tÐQVÝ�9¨$Ð/Ñ/Ô/ðñ ô €Jõ
 'ÐvÐvÈZÐvÕ[_Ð`jÐ[uÐ[uÐntÐ[uÐ[uÐvÐv€EØ€LrU   c                 ó”   — t          ddddddddddt          t          d	¬
¦  «        ¬¦  «        }t          dd| it          |fi |¤Ž¤Ž}|S )z} BEiT3 Giant model with patch size 14x14 and image size 336x336.
    Remapped to VisionTransformer with scale_norm=True.
    rÄ  r  r  r!  r(  r¢  Tr   râ   rB  )r-  r.  r5  rJ   rB   rY   rZ   r[   r6  r1  rI   Úbeit3_giant_patch14_336rØ  )r¥  rE  rï  s       rS   r¥  r¥  ”  sr   € õ
 Ø ¨t¸2ÈÐW]Ø¨T¸tÐQVÝ�9¨$Ð/Ñ/Ô/ðñ ô €Jõ
 'ÐvÐvÈZÐvÕ[_Ð`jÐ[uÐ[uÐntÐ[uÐ[uÐvÐv€EØ€LrU   Úvit_tiny_patch16_224_in21kÚvit_small_patch32_224_in21kÚvit_small_patch16_224_in21kÚvit_base_patch32_224_in21kÚvit_base_patch16_224_in21kÚvit_base_patch8_224_in21kÚvit_large_patch32_224_in21kÚvit_large_patch16_224_in21kÚvit_huge_patch14_224_in21kÚvit_base_patch32_224_samzvit_base_patch32_224.samÚvit_base_patch16_224_samzvit_base_patch16_224.samÚvit_small_patch16_224_dinoÚvit_small_patch8_224_dinoÚvit_base_patch16_224_dinoÚvit_base_patch8_224_dinoÚvit_base_patch16_224_miil_in21kÚ!vit_base_patch32_224_clip_laion2b)Ú"vit_large_patch14_224_clip_laion2bÚ!vit_huge_patch14_224_clip_laion2bÚ"vit_giant_patch14_224_clip_laion2b)FFFTr?   r?   Nr   )r  r0   Frâ  )r<   r?   T)rH  r?   T)r0   rj   rö  F)r<   F)rf  )Frö  Trã  )FNrä  (  rˆ   ÚcopyÚloggingrð   ÚosÚcollectionsr   Ú	functoolsr   Útypingr   r   r   r   r	   r
   r   r   r   r   ÚImportErrorÚtyping_extensionsr�   Útorch.nnrn   Útorch.nn.functionalÚ
functionalrË   Ú	torch.jitr   Ú	timm.datar   r   r   r   r   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   Ú_builderr1   Ú	_featuresr2   Ú_helpersr3   Ú_manipulater4   r5   r6   r7   Ú	_registryr8   r9   r:   Ú__all__Ú	getLoggerr…   Ú_loggerrO   rs   r‹   rŠ   rŒ   rT   rW   r’   rž   rÞ   r  rŽ   rN   r$  r;   rŽ  rí  rò  rô  rŠ  r   rz  r“  r{  r…  r‹  r_  rž  r®  r¹  Údefault_cfgsru  Ú_quick_gelu_cfgsr×  ÚdeepcopyrÑ  rv  ÚenvironrP   Úlowerrå  ré  rí  ró  r÷  rú  rý  rÿ  r  r  r  r	  r  r  r  r  r  r  r  r  r#  r(  r*  r-  r0  r2  r5  r8  r:  r=  r@  rD  rI  rK  rM  rP  rS  rU  rW  rY  r[  r]  r_  rb  rd  rg  ri  rl  ro  rr  rv  ry  r{  r~  r€  rƒ  r…  rˆ  r‹  r�  r�  r”  r—  rš  rœ  rŸ  r¡  r£  r§  rª  r¬  r®  r°  r²  r·  r¼  r¾  rÀ  rÃ  rÅ  rÇ  rË  rÍ  rÑ  rÔ  r×  rÙ  rÛ  rÝ  rà  râ  ræ  ré  rë  rî  rñ  ró  rõ  rø  rû  rþ  r  r  r  r  r
  r  r  r  r  r  r  r  r  r  r!  r$  r(  r+  r.  r0  r4  r7  r9  r<  r?  rB  rE  rG  rJ  rL  rN  rR  rU  rX  r`  rc  re  rg  rm  rs  rw  r|  r  r�  r„  r†  rˆ  rŠ  rŒ  rŽ  r’  r”  r˜  r›  rž  r   r£  r¥  rj   rU   rS   ú<module>rÕ     s"a  ððð ð2 €€€Ø €€€Ø €€€Ø 	€	€	€	Ø #Ð #Ð #Ð #Ð #Ð #Ø Ð Ð Ð Ð Ð Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ Oð*ØÐÐÐÐÐÐøØð *ð *ð *Ø)Ð)Ð)Ð)Ð)Ð)Ð)Ð)ð*øøøð €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð
ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð8 +Ð *Ð *Ð *Ð *Ð *Ø +Ð +Ð +Ð +Ð +Ð +Ø *Ð *Ð *Ð *Ð *Ð *Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YàÐ
€ð ˆ'Ô
˜HÑ
%Ô
%€ð 	ØØðð €ð ØØ ØØØØ04Øñ!ð !Øð!àð!ð ð!ð ð	!ð
 ð!ð ð!ð ð!ð ð!ð ð!ð ˜T "¤)œ_Ô-ð!ð ð!ð „Yð!ð !ð !ð !ðHVð Vð Vð Vð VˆBŒIñ Vô Vð VðrHð Hð Hð Hð H�2”9ñ Hô Hð HðV@ð @ð @ð @ð @˜2œ9ñ @ô @ð @ðFkð kð kð kð k˜rœyñ kô kð kð\Vð Vð Vð Vð V˜"œ)ñ Vô Vð Vðv !Ø!"Ø&+ñ	ð ØŒ<ðàðð ðð  $ð	ð ð ð ð2T	ð T	ð T	ð T	ð T	˜œ	ñ T	ô T	ð T	ñn"ð " "¤)ð "°3ð "È$ð "ÐZ^ð "ð "ð "ð "ð( ØØ ñ	"ð "Ø”	ð"àð"ð ð"ð ð	"ð
 
ð"ð "ð "ð "ñ@"ð " "¤)ð "°3ð "È$ð "ÐZ^ð "ð "ð "ð "ñ."ð "¨"¬)ð "¸3ð "ÐRVð "Ðbfð "ð "ð "ð "ñ

Gð 
G˜sð 
G°uð 
GÐQUð 
GÐaið 
Gð 
Gð 
Gð 
Gð  "#Ø"$Ø&Øñð Ø”ðà”Lðð ðð �c˜3�h”ð	ð
 ðð ðð „\ðð ð ð ð2 €„�„ñLTð LT˜œð LT°Sð LTÀ#ð LTÐ[_ð LTÐlpð LTð LTð LTñ „ðLTðd  ñ#ð #Ø˜˜eœlÐ*Ô+ð#à ð#ð ð#ð 
ˆ#ˆuŒ|Ð
Ôð	#ð #ð #ð #ðLØ˜˜eœlÐ*Ô+ðà ðð 
ˆ#ˆuŒ|Ð
Ôðð ð ð ð.Ø˜˜eœlÐ*Ô+ðà ðð 
ˆ#ˆuŒ|Ð
Ôðð ð ð ð$9˜tð 9ð 9ð 9ð 9ð~ #(Ø&ØñQð QØ˜˜eœlÐ*Ô+ðQà ðQð  ðQð ð	Qð
 ðQð 
ˆ#ˆuŒ|Ð
ÔðQð Qð Qð Qñhð ˆcð  T¨#¨s¨(¤^ð ð ð ð ð"Sð 1°$°$Øð3ñ 3ô 3ðSð
 0°°Øð2ñ 2ô 2ðSð 0°°ð dØØð2ñ 2ô 2ðSð 0°°ð dØØ ]¸Sð2Bñ 2Bô 2BðSð" 1°$°$ð eØØð3ñ 3ô 3ð#Sð* 1°$°$ð eØØ ]¸Sð3Bñ 3Bô 3Bð+Sð2 1°$°$ð eØØð3ñ 3ô 3ð3Sð: 1°$°$ð eØØ ]¸Sð3Bñ 3Bô 3Bð;SðB 0°°ð fØØð2ñ 2ô 2ðCSðJ 0°°ð dØØ ]¸Sð2Bñ 2Bô 2BðKSðR 0°°ð eØØð2ñ 2ô 2ðSSðZ 0°°ð eØØ ]¸Sð2Bñ 2Bô 2Bð[Sðb /°°ð dØØð1ñ 1ô 1ðcSðj 1°$°$ð eØØð3ñ 3ô 3ðkSðr 1°$°$ð eØØ ]¸Sð3Bñ 3Bô 3BðsSð~ .¨t¨tØ}Øð0ñ 0ô 0ðSðD .¨t¨tØ}ØØ ¨3ð00ñ 00ô 00ðESð SðL /°°Ø~ØØ ¨3ð10ñ 10ô 10ðMSðX (¨¨ð dØØð*ñ *ô *ðYSð` (¨¨ð dØØ ]¸Sð*Bñ *Bô *BðaSðh '¨¨ð dØØð)ñ )ô )ðiSðp '¨¨ð dØØ ]¸Sð)Bñ )Bô )BðqSðx '¨¨ð dØØð)ñ )ô )ðySð@ '¨¨ð dØØ ]¸Sð)Bñ )Bô )BðASðJ & t t° |¡|¤|ðKSðL % d d¨r l¡l¤lðMSðN & t t° |¡|¤|ðOSðP )¨$¨$°2¨,©,¬,ðQSðV & t tàØð(ñ (ô (ðWSð^ & t tàØð(ñ (ô (ð_Sðf '¨¨àØð)ñ )ô )ðgSðn '¨¨àØð)ñ )ô )ðoSðv & t tØØð(ñ (ô (ðwSð@ (¨¨Ø{ØØ eð*-ñ *-ô *-ðASð Sð SðH )¨$¨$Ø|ØØ eð+-ñ +-ô +-ðISðP )¨$¨$Ø|ØØ eð+-ñ +-ô +-ðQSðX (¨¨Ø}ØØ eð*-ñ *-ô *-ðYSð` (¨¨Ø|ØØ eð*-ñ *-ô *-ðaSðh '¨¨Ø{ØØ eð)-ñ )-ô )-ðiSðp )¨$¨$Ø|ØØ eð+-ñ +-ô +-ðqSð| $ T TØHÐVZØð&ñ &ô &ð}SðB $ T TØHÐVZØð&ñ &ô &ðCSðL ! $ $ØiØØ"Ð(<È!ð#Mñ #Mô #MðMSðT    ØgØØ"Ð(<È!ð"Mñ "Mô "MðUSð\    ØeØØ"Ð(<È!ð"Mñ "Mô "Mð]Sðd   ØcØØ"Ð(<È!ð!Mñ !Mô !MðeSðp '¨¨Ø\ØØØ"Ð(<È!Ø ¨3ð)0ñ )0ô )0ðqSð| & t tØ\ØØØ"Ð(<È!Ø ¨3ð(0ñ (0ô (0ð}SðH '¨¨Ø\ØØØ"Ð(<È!Ø ¨3ð)0ñ )0ô )0ðISðT '¨¨Ø\ØØØ"Ð(<È!Ø ¨3ð)0ñ )0ô )0ðUSðd ,¨T¨TØaØØØ"Ð(<È!Ø ¨3ð.0ñ .0ô .0ðeSð Sð Sðp +¨D¨DØaØØØ"Ð(<È!Ø ¨3ð-0ñ -0ô -0ðqSð| ,¨T¨TØaØØØ"Ð(<È!Ø ¨3ð.0ñ .0ô .0ð}SðH ,¨T¨TØaØØØ"Ð(<È!Ø ¨3ð.0ñ .0ô .0ðISðZ )¨$¨$ØØØ˜|¸Ø ¨3ð	+0ñ +0ô +0ð[Sðd *¨4¨4ØØØ˜|¸Ø ¨3ð	,0ñ ,0ô ,0ðeSðn +¨D¨DØØØ˜|¸Ø ¨3ð	-0ñ -0ô -0ðoSðx *¨4¨4ØØØ˜|¸Ø ¨3ð	,0ñ ,0ô ,0ðySðF & t tð LØØ˜|°eÈ:Ðchð(jñ (jô (jðGSðN .¨t¨tð NØØ˜|°eÈ:ð0Wñ 0Wô 0WðOSðZ '¨¨ð LØð)ñ )ô )ð[Sð` *¨4¨4ØØ ¨4¸Uð,Dñ ,Dô ,DðaSðf 2°4°4ØØ ¨4ð41ñ 41ô 41ðgSðl 2°4°4ØØ ¨4¸8ð4Eñ 4Eô 4EðmSðr ,¨T¨TØ ¨4ð.1ñ .1ô .1ðsSðv )¨$¨$©&¬&ðwSð| 6°t°tØØ ?ð84ñ 84ô 84ð}SðB	 6°t°tØØ ?¸SÈ]ð8\ñ 8\ô 8\ðC	Sð Sð SðH	 6°t°tØØ ?¸SÈ]ð8\ñ 8\ô 8\ðI	SðN	 6°t°tØØ ?¸Tð8Cñ 8Cô 8CðO	SðT	 6°t°tØØ ?Ø ¸(ð8Dñ 8Dô 8DðU	Sð\	 7¸¸ØØ$Ð*@È3ð9Pñ 9Pô 9Pð]	Sðb	 7¸¸ØØ$Ð*@Ø ¸(ð9Dñ 9Dô 9Dðc	Sðj	 6°t°tØØ ?¸Sð8Bñ 8Bô 8Bðk	Sðp	 6°t°tØØ ?Ø ¸(ð8Dñ 8Dô 8Dðq	Sðz	 5°d°dà ?ð74ñ 74ô 74ð{	Sð@
 5°d°dØØ ?Ø -¸8ð7Eñ 7Eô 7EðA
SðH
 5°d°dØØ ?¸Tð7Cñ 7Cô 7CðI
SðN
 5°d°dØØ ?Ø -¸8ð7Eñ 7Eô 7EðO
SðV
 6°t°tØØ ?¸Sð8Bñ 8Bô 8BðW
Sð\
 6°t°tØØ ?Ø ¸(ð8Dñ 8Dô 8Dð]
Sðf
 0°°ØØ ?ð24ñ 24ô 24ðg
Sðl
 0°°ØØ ?¸Sð2Bñ 2Bô 2Bðm
Sñr
 0°°ØØ ?Ø ¸(ð2Dñ 2Dô 2Dðs
Sñz
 1°$°$ØØ$Ð*@È3ð3Pñ 3Pô 3Pð{
Sð Sð Sñ@ 1°$°$ØØ$Ð*@Ø ¸(ð3Dñ 3Dô 3DðASñH 0°°ØØ ?¸Sð2Bñ 2Bô 2BðISñN 0°°ØØ ?Ø ¸(ð2Dñ 2Dô 2DðOSñX /°°ØØ ?ð14ñ 14ô 14ðYSñ^ /°°ØØ ?ð14ñ 14ô 14ð_Sñd /°°ØØ ?Ø ¸(ð1Dñ 1Dô 1DðeSñl 0°°ØØ ?¸Sð2Bñ 2Bô 2BðmSñt 1°$°$ØØ ?Àñ3Gñ 3Gô 3GðuSñz 2°4°4ØØ$Ð*@È3Ð\añ4cñ 4cô 4cð{Sñ@ 1°$°$ØØ ?¸SÈeñ3Uñ 3Uô 3UðASñH 0°°ØØ ?Àñ2Gñ 2Gô 2GðISñN 1°$°$ØØ ?¸SÈeñ3Uñ 3Uô 3UðOSñV (¨¨ØØ ?Áñ*Eñ *Eô *EðWSñ\ (¨¨ØØ ?¸SÉcñ*Sñ *Sô *Sð]Sñb )¨$¨$ØØ$Ð*@È3Ñ\_ñ+añ +aô +aðcSñh (¨¨ØØ ?¸SÉdñ*Tñ *Tô *TðiSñn )¨$¨$ØØ ?¸SÉdñ+Tñ +Tô +TðoSð Sð Sñt ,¨T¨TØØ ?¸SÉdñ.Tñ .Tô .TðuSñ| .¨t¨tØÙÙWØ ?Áñ	0Eñ 0Eô 0Eð}SñF .¨t¨tØÙØ ?¸SÉcñ0Sñ 0Sô 0SðGSñN 3°D°DØÙØ ?Ø ¨3¹Cñ	5Añ 5Aô 5AðOSñX /°°ØÙØ ?¸SÉcñ1Sñ 1Sô 1SðYSñb +¨D¨DØØ ?¸SÉcñ-Sñ -Sô -SðcSñh +¨D¨DØØ ?Ø ¹Cñ-Añ -Aô -AðiSñp +¨D¨DØØ ?¸SÉcñ-Sñ -Sô -SðqSñv ,¨T¨TØØ ?¸SÉcñ.Sñ .Sô .SðwSñ~ & t tØÙØ ?¸SÉcñ(Sñ (Sô (SðSñF ,¨T¨TØÙØ ?¸SÉcñ.Sñ .Sô .SðGSñN '¨¨ØÙÙWØ ?¸SÉcñ	)Sñ )Sô )SðOSñX & t tØÙÙWØ ?¸SÉdñ	(Tñ (Tô (TðYSñb & t tØØ ?ÙÙWØ¡¹Dñ(Bñ (Bô (BðcSñz 4°T°TØÙÙWØ ?¸SÉdñ	6Tñ 6Tô 6Tð{SñD 4°T°TØÙØ ?Ù ¨3¸(ÑPTñ	6Vñ 6Vô 6VðESñN 8¸¸ØÙØ ?¸SÉdñ:Tñ :Tô :TðOSð Sð SñV 8¸¸ØÙØ ?Ù ¨3¸(ÑPTñ	:Vñ :Vô :VðWSñb /°°ØÙÙWØ ?¸SÉcñ	1Sñ 1Sô 1SðcSñl /°°ØÙÙWØ ?¸SÉcñ	1Sñ 1Sô 1SðmSñv 0°°ØÙÙWØ ?¸SÉcñ	2Sñ 2Sô 2SðwSñ@ /°°ØÙÙWØ ?¸SÉdñ	1Tñ 1Tô 1TðASñJ 4°T°TØÙØ ?¸SÉdñ6Tñ 6Tô 6TðKSñR 3°D°DØÙÙWØ ?¸SÉdñ	5Tñ 5Tô 5TðSSñ\ .¨t¨tØÙÙWØ ?¸SÉcñ	0Sñ 0Sô 0Sð]Sñf .¨t¨tØÙÙWØ ?¸SÉcñ	0Sñ 0Sô 0SðgSñp /°°ØÙÙWØ ?¸SÉcñ	1Sñ 1Sô 1SðqSñ| '¨¨ØÙWØ ?Áñ)Eñ )Eô )Eð}SñD '¨¨ØÙWØ ?Áñ)Eñ )Eô )EðESñL (¨¨ØÙWØ ?¸SÉcñ*Sñ *Sô *SðMSñT (¨¨ØÙWØ ?Ø ¹Cñ	*Añ *Aô *AðUSñ` 6°t°tØÙØ ?¸SÙñ	8ñ 8ô 8ðaSñp *¨4¨4°BÀ=Ð[_Ñ+`Ñ+`Ô+`ðqSñr *¨4¨4°BÀ=Ð[_Ñ+`Ñ+`Ô+`ðsSð Sð Sñt +¨D¨D°R¨L©L¬LðuSñv +¨D¨D°R¨L©L¬LðwSñx *¨4¨4°B¨<©<¬<ðySñ@ 0°°à¡5Ø ?Ù ¨3ñ	20ñ 20ô 20ðASñJ 0°°à¡5Ø ?Ø ¨3¸(ñ	2Dñ 2Dô 2DðKSñT *¨4¨4à¡5Ø ?Ù ¨3ñ	,0ñ ,0ô ,0ðUSñ^ *¨4¨4à¡5Ø ?Ø ¨3¸(ñ	,Dñ ,Dô ,Dð_Sñj ! $ $ÙSÐaeØØ ¨4ñ#1ñ #1ô #1ðkSñr    ÙYÐgkØØ ¨4ñ"1ñ "1ô "1ðsSñz    ÙYÐgkØØ ¨4ñ"1ñ "1ô "1ð{SñD    ÙSÐaeØØ ¨4ñ"1ñ "1ô "1ðESñL   ÙYÐgkØØ ¨4ñ!1ñ !1ô !1ðMSñT   ÙYÐgkØØ ¨4ñ!1ñ !1ô !1ðUSñ\ ! $ $Ù[ÐimØØ ¨4¸Uñ#Dñ #Dô #Dð]Sñd    ÙZÐhlØØ ¨4¸Uñ"Dñ "Dô "DðeSñn ! $ $ÙSÐaeØØ ¨4ñ#1ñ #1ô #1ðoSñv    ÙYÐgkØØ ¨4ñ"1ñ "1ô "1ðwSð Sð Sñ~    ÙYÐgkØØ ¨4ñ"1ñ "1ô "1ðSñH " 4 4ÙWÐeiØØ ¨4¸Uñ$Dñ $Dô $DðISñP " 4 4ÙWÐeiØØ ¨4¸Uñ$Dñ $Dô $DðQSñZ (¨¨°"¨©¬ð[Sñ\ )¨$¨$°2¨,©,¬,ð]Sñ^ (¨¨°"¨©¬ð_Sñb   ÙSØÙØ"Ð(<È!ñ	!Mñ !Mô !MðcSñl    ÙTØÙØ"Ð(<È!ñ	"Mñ "Mô "MðmSñv   ÙSØÙØ"Ð(<È!ñ	!Mñ !Mô !MðwSñB *¨4¨4ÙMáØ"Ð(<È!ñ	,Mñ ,Mô ,MðCSñL +¨D¨DÙNáØ"Ð(<È!ñ	-Mñ -Mô -MðMSñV *¨4¨4ÙSáØ ¨3Ø"Ð(<È!ñ,Mñ ,Mô ,MðWSñb ,¨T¨TÙNáØ"Ð(<È!ñ	.Mñ .Mô .MðcSñn +¨D¨DØØ Øñ-ñ -ô -ðoSñv +¨D¨DØØð-ñ -ô -ðwSñ| (¨¨ØØð*ñ *ô *ð}SñB +¨D¨DØØ Øñ-ñ -ô -ðCSð Sð SñJ (¨¨ØØ Øñ*ñ *ô *ðKSñR -¨d¨dØØ Øñ/ñ /ô /ðSSñZ +¨D¨DØØ Øñ-ñ -ô -ð[Sñb (¨¨ØØ Øñ*ñ *ô *ðcSñj +¨D¨DØÙ Øñ-ñ -ô -ðkSñr (¨¨ØÙ Øñ*ñ *ô *ðsSñz ,¨T¨TØØ Øñ.ñ .ô .ð{SñB )¨$¨$ØØ Øñ+ñ +ô +ðCSñJ ,¨T¨TØØ Øñ.ñ .ô .ðKSñR )¨$¨$ØØ Øñ+ñ +ô +ðSSñZ ,¨T¨TØÙ Øñ.ñ .ô .ð[Sñb -¨d¨dØØð/ñ /ô /ðcSñh *¨4¨4ØØð,ñ ,ô ,ðiSñn -¨d¨dØÙ Øñ/ñ /ô /ðoSñv *¨4¨4ØÙ Øñ,ñ ,ô ,ðwSñ~ *¨4¨4ØØ Øñ,ñ ,ô ,ðSñF -¨d¨dØØ Øñ/ñ /ô /ðGSð Sð SñN /°°ØØ Øñ1ñ 1ô 1ðOSñV -¨d¨dØØ Øñ/ñ /ô /ðWSñ^ -¨d¨dØÙ Øñ/ñ /ô /ð_Sñf /°°ØØ Øñ1ñ 1ô 1ðgSñn /°°ØØ Øñ1ñ 1ô 1ðoSñx /°°ØØ Øñ1ñ 1ô 1ðySñ@ /°°ØØð1ñ 1ô 1ðASñF ,¨T¨TØØð.ñ .ô .ðGSñL /°°ØØ Øñ1ñ 1ô 1ðMSñT ,¨T¨TØØ Øñ.ñ .ô .ðUSñ\ 1°$°$ØØ Øñ3ñ 3ô 3ð]Sñd /°°ØØ Øñ1ñ 1ô 1ðeSñl ,¨T¨TØØ Øñ.ñ .ô .ðmSñt /°°ØÙ Øñ1ñ 1ô 1ðuSñ| ,¨T¨TØÙ Øñ.ñ .ô .ð}SñD 0°°ØØ Øñ2ñ 2ô 2ðESñL -¨d¨dØØ Øñ/ñ /ô /ðMSð Sð SñT 0°°ØØ Øñ2ñ 2ô 2ðUSñ\ -¨d¨dØØ Øñ/ñ /ô /ð]Sñd 0°°ØÙ Øñ2ñ 2ô 2ðeSñl 1°$°$ØØð3ñ 3ô 3ðmSñr .¨t¨tØØð0ñ 0ô 0ðsSñx 1°$°$ØØð3ñ 3ô 3ðySñ~ 0°°ØØð2ñ 2ô 2ðSñD 4°T°TØØð6ñ 6ô 6ðESñJ 5°d°dØØð7ñ 7ô 7ðKSñZ 1°$°$ØÙ Øñ3ñ 3ô 3ð[Sñb .¨t¨tØÙ ¨3Øð0ñ 0ô 0ðcSñj .¨t¨tØØ ¨3Øð0ñ 0ô 0ðkSñr 1°$°$ØØ ¨3Øð3ñ 3ô 3ðsSñz 0°°ØØ ¨3Øð2ñ 2ô 2ð{SñB 9¸$¸$ØØ ¨3Øð;ñ ;ô ;ðCSñJ 4°T°TØØ ¨3Øð6ñ 6ô 6ðKSñR 4°T°TØØ ¨3Øð6ñ 6ô 6ðSSð Sð SñZ 5°d°dØØ ¨3Øð7ñ 7ô 7ð[Sñn 7¸¸ØØ ¨3Øð9ñ 9ô 9ðoSñv 8¸¸ØØ ¨3Øð:ñ :ô :ðwSñ~ 0°°ØÙ ¨3Øð2ñ 2ô 2ðSñF 9¸$¸$ØÙ ¨3Øð;ñ ;ô ;ðGSñN 4°T°TØÙ ¨3Øð6ñ 6ô 6ðOSñV 4°T°TØÙ ¨3Øð6ñ 6ô 6ðWSñ^ 5°d°dØÙ ¨3Øð7ñ 7ô 7ð_Sñr 1°$°$ØØ Øñ3ñ 3ô 3ðsSñz 3°D°DØØ Øñ5ñ 5ô 5ð{SñB 1°$°$ØØ Øñ3ñ 3ô 3ðCSñJ 1°$°$ØÙ Øñ3ñ 3ô 3ðKSñR 3°D°DØØ Øñ5ñ 5ô 5ðSSñZ 3°D°DØØ Øñ5ñ 5ô 5ð[Sñd 2°4°4ØÙ ¨3¸(ð4ñ 4ô 4ðeSñl 6°t°tØÙ ¨3¸(ð8ñ 8ô 8ðmSñv 3°D°DØÙØ ?Áñ5Eñ 5Eô 5EðwSð Sð Sñ~ 5°d°dØÙØ ?Áñ7Eñ 7Eô 7EðSñF 3°D°DØÙØ ?Áñ5Eñ 5Eô 5EðGSñN 6°t°tØÙØ ?Áñ8Eñ 8Eô 8EðOSñX ,¨T¨TØØ ¨4ð.1ñ .1ô .1ðYSñ^ 6°t°tØØ ¨4ð81ñ 81ô 81ð_Sñd -¨d¨dØØ ¨4ð/1ñ /1ô /1ðeSñj -¨d¨dØØ ¨4ð/1ñ /1ô /1ðkSñp 7¸¸ØØ ¨4ð91ñ 91ô 91ðqSñv .¨t¨tØØ ¨4ð01ñ 01ô 01ðwSñ| 8¸¸ØØ ¨4ð:1ñ :1ô :1ð}SñB  0°°ØØØ ¨4ñ21ñ 21ô 21ðC SñJ  /°°ØØ ¨4ð11ñ 11ô 11ðK SñP  9¸$¸$ØØ ¨4ð;1ñ ;1ô ;1ðQ SñV  /°°ØØ ¨4ð11ñ 11ô 11ðW Sñ\  8¸¸ØØ ¨4ð:1ñ :1ô :1ð] Sñb  /°°ØØ ¨4ð11ñ 11ô 11ðc Sñh  0°°ØØØ ¨4ñ21ñ 21ô 21ði Sð Sð Sñp  ?ÀÀØØ ¨4ðA1ñ A1ô A1ðq Sñv  9¸$¸$ØØ ¨4ð;1ñ ;1ô ;1ðw Sñ|  7¸¸ØØØ ¨4ñ91ñ 91ô 91ð} SñD! 1°$°$ØØØ ¨4ñ31ñ 31ô 31ðE!SñL! ?ÀÀØØ ¨3ðA0ñ A0ô A0ðM!SñR! 0°°ØØ ¨4ð21ñ 21ô 21ðS!SñX! ?ÀÀØØ ¨4ðA1ñ A1ô A1ðY!Sñ^! 9¸$¸$ØØ ¨4ð;1ñ ;1ô ;1ð_!Sñd! 0°°ØØ ¨4ð21ñ 21ô 21ðe!Sñj! 7¸¸ØØØ ¨4ñ91ñ 91ô 91ðk!Sñr! 1°$°$ØØØ ¨4ñ31ñ 31ô 31ðs!Sñz! ?ÀÀØØ ¨3ðA0ñ A0ô A0ð{!Sñ@" .¨t¨tØ ñ0"ñ 0"ô 0"ðA"SñF" =¸d¸dØØ ¨4ð?1ñ ?1ô ?1ðG"SñL" 5°d°dØØØ ¨4ñ71ñ 71ô 71ðM"SñT" =¸d¸dØØ ¨3ð?0ñ ?0ô ?0ðU"SñZ" 0°°Ø ñ2"ñ 2"ô 2"ð["Sð Sð Sñ^" >¸t¸tØØ ¨3ð@0ñ @0ô @0ð_"Sñd" 6°t°tØØØ ¨3ñ80ñ 80ô 80ðe"Sñl" >¸t¸tØØ ¨3ð@0ñ @0ô @0ðm"Sñr" >¸t¸tØØ ¨3¸(ð@Dñ @Dô @Dðs"Sñz" +¨D¨DØÙØ"Ð(<Ø ¨3¸Añ	-ñ -ô -ð{"SñF# +¨D¨DØØ"Ð(<Ø ¨3¸Añ-ñ -ô -ðG#SñR# '¨¨ØØ ?¹LØ !ñ)%ñ )%ô )%ðS#SñZ# ,¨T¨TØØ ?¹LØ !ñ.%ñ .%ô .%ð[#Sñb# & t tØØ ?¹LØ !ñ(%ñ (%ô (%ðc#Sñj# $ T TØØ ?¹LØ !ñ&%ñ &%ô &%ðk#Sñr# $ T TØØ ?¹LØ !ñ&%ñ &%ô &%ðs#Sñz# '¨¨ØØ ?¹LØ ¨3¸Añ)?ñ )?ô )?ð{#SñB$ ,¨T¨TØØ ?¹LØ ¨3¸Añ.?ñ .?ô .?ðC$SñJ$ & t tØØ ?¹LØ ¨3¸Añ(?ñ (?ô (?ðK$SñR$ $ T TØØ ?¹LØ ¨3¸Añ&?ñ &?ô &?ðS$SñZ$ $ T TØØ ?¹LØ ¨3¸Añ&?ñ &?ô &?ð[$Sñb$ '¨¨ØØ ?¹LØ ¨3¸Añ)?ñ )?ô )?ðc$Sð Sð Sñj$ & t tØØ ?¹LØ ¨3¸Añ(?ñ (?ô (?ðk$Sñr$ $ T TØØ ?¹LØ ¨3¸Añ&?ñ &?ô &?ðs$Sñz$ $ T TØØ ?¹LØ ¨3¸Añ&?ñ &?ô &?ð{$SñD% ˜$˜$ØÙ ¨4ð1ñ 1ô 1ðE%SñJ% ˜4˜4ØÙ ¨4ð1ñ 1ô 1ðK%SñP% ˜4˜4ØÙ ¨4ð1ñ 1ô 1ðQ%SñV% ˜4˜4Ù ¨4ð1ñ 1ô 1ðW%Sñ^% +¨D¨DØØ"Ð(<Àsð-Lñ -Lô -Lð_%Sñd% 4°T°TØØ"Ð(<Àsð6Lñ 6Lô 6Lðe%Sñj% ,¨T¨TØØ"Ð(<Àsð.Lñ .Lô .Lðk%Sñp% 5°d°dØØ"Ð(<Àsð7Lñ 7Lô 7Lðq%Sñv% (¨¨ØÐ*Ð0DÈsñ*Tñ *Tô *Tðw%Sñz% (¨¨Ø˜=Ð/DÐJ^Ðilñ*nñ *nô *nð{%Sñ~%    ØØ"Ð(<ÀsØñ"ñ "ô "ð%SñH& )¨$¨$ØØ"Ð(<ÀsØñ+ñ +ô +ðI&SñR& ! $ $ØØ"Ð(<ÀsØñ#ñ #ô #ðS&Sñ\& *¨4¨4ØØ"Ð(<ÀsØñ,ñ ,ô ,ð]&Sð S€ñj& oÐn ,×"4Ò"4Ñ"6Ô"6ÐnÑnÔnÐ Ø	ð >ð >€AàˆŒ�l 1”oÑ&Ô&€AØ‰„~˜Ò Ð Ø  1™ˆ‰+‰Ø<=€L�—’™8Ñ%7Ñ8Ô8Ñ9Ð9Ø$Ð$ \Ñ2Ô2€ð ”j—n’nÑ%9¹7ÑCÔC×IÒIÑKÔKÉvÒUÐ ð !Ø%)ñ#ñ #Øñ#àñ#ð ˜T”Nð#ð
 Ð™kÐ)Ô*ð#ñ #ð #ð #ðL ññ  Tð Ð@Qð ñ ð ñ „ðð ññ  Tð Ð@Qð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  Tð Ð@Qð ñ ð ñ „ðð ññ  Tð Ð@Qð ñ ð ñ „ðð ññ  Tð Ð@Qð ñ ð ñ „ðð ññ  Tð Ð@Qð ñ ð ñ „ðð ññ  Tð Ð@Qð ñ ð ñ „ðð ññ  Dð Ð?Pð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  Tð Ð@Qð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ ¨ð ÐDUð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨4ð ÐFWð ñ ð ñ „ðð ññ ¨4ð ÐFWð ñ ð ñ „ðð ññ ¨4ð ÐFWð ñ ð ñ „ðð ññ ¨Dð ÐGXð ñ ð ñ „ðð ññ ¨ð ÐDUð ñ ð ñ „ðð ññ ¨ð ÐDUð ñ ð ñ „ðð ññ ¨ð ÐDUð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨Dð ÐGXð ñ ð ñ „ðð ññ ¨Dð ÐGXð ñ ð ñ „ðð ññ ¨Dð ÐGXð ñ ð ñ „ðð ññ ¨Tð ÐHYð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨tð ÐJ[ð ñ ð ñ „ðð ññ ¨4ð ÐFWð ñ ð ñ „ðð ññ ¨4ð ÐFWð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ñ
ñ 
¨4ð 
ÐFWð 
ñ 
ð 
ñ „ð
ð ñ
ñ 
¨dð 
ÐIZð 
ñ 
ð 
ñ „ð
ð ñ
ñ 
¨dð 
ÐIZð 
ñ 
ð 
ñ „ð
ð ñ	ñ 	°Dð 	ÐO`ð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	°Dð 	ÐO`ð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	°Tð 	ÐPað 	ñ 	ð 	ñ „ð	ð ñ	ñ 	°Tð 	ÐPað 	ñ 	ð 	ñ „ð	ð ñ	ñ 	°Dð 	ÐO`ð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	°Dð 	ÐO`ð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	¸ð 	ÐSdð 	ñ 	ð 	ñ „ð	ð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ ¨ð ÐDUð ñ ð ñ „ðð ññ ¨4ð ÐFWð ñ ð ñ „ðð ñ	ñ 	¨4ð 	ÐFWð 	ñ 	ð 	ñ „ð	ð ññ ¨$ð ÐEVð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ  dð ÐARð ñ ð ñ „ðð ññ ˜tð Ð:Kð ñ ð ñ „ðð ññ ˜dð Ð9Jð ñ ð ñ „ðð ññ ˜tð Ð:Kð ñ ð ñ „ðð ñ	ñ 	¨ð 	ÐCTð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	¨ð 	ÐDUð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	¨ð 	ÐCTð 	ñ 	ð 	ñ „ð	ð ññ ¨ð ÐDUð ñ ð ñ „ðð ññ ¨ð ÐCTð ñ ð ñ „ðð ññ ¨ð ÐDUð ñ ð ñ „ðð ññ ¨ð ÐDUð ñ ð ñ „ðð  ñ	ñ 	¨dð 	ÐIZð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	¨Tð 	ÐHYð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	¨dð 	ÐIZð 	ñ 	ð 	ñ „ð	ð ññ ¨dð ÐIZð ñ ð ñ „ðð  ñ	ñ 	¨Tð 	ÐHYð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	¨dð 	ÐIZð 	ñ 	ð 	ñ „ð	ð ñ	ñ 	¨tð 	ÐJ[ð 	ñ 	ð 	ñ „ð	ð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ ¨Dð ÐGXð ñ ð ñ „ðð ññ ¨Dð ÐGXð ñ ð ñ „ðð ññ ¨Dð ÐGXð ñ ð ñ „ðð ññ ¨Dð ÐGXð ñ ð ñ „ðð ññ ¨Dð ÐGXð ñ ð ñ „ðð ññ ¨Tð ÐHYð ñ ð ñ „ðð ññ ¨Tð ÐHYð ñ ð ñ „ðð ññ ¨Tð ÐHYð ñ ð ñ „ðð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐL]ð ñ ð ñ „ðð ññ °ð ÐL]ð ñ ð ñ „ðð ññ °ð ÐL]ð ñ ð ñ „ðð ññ °$ð ÐM^ð ñ ð ñ „ðð ññ °$ð ÐM^ð ñ ð ñ „ðð ññ °$ð ÐM^ð ñ ð ñ „ðð ññ °$ð ÐM^ð ñ ð ñ „ðð ññ °$ð ÐM^ð ñ ð ñ „ðð ññ °$ð ÐM^ð ñ ð ñ „ðð ññ °$ð ÐM^ð ñ ð ñ „ðð ññ °$ð ÐM^ð ñ ð ñ „ðð ññ °Dð ÐO`ð ñ ð ñ „ðð ññ °Dð ÐO`ð ñ ð ñ „ðð ññ ¨Tð ÐHYð ñ ð ñ „ðð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ ¨tð ÐJ[ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐL]ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐL]ð ñ ð ñ „ðð ññ °ð ÐL]ð ñ ð ñ „ðð ññ °ð ÐL]ð ñ ð ñ „ðð ññ °ð ÐL]ð ñ ð ñ „ðð ññ °ð ÐL]ð ñ ð ñ „ðð ññ ¨dð ÐIZð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ðð ññ °ð ÐK\ð ñ ð ñ „ñð ññ °ð ÐK\ð ñ ð ñ „ñð ññ °ð ÐL]ð ñ ð ñ „ñð ññ °ð ÐL]ð ñ ð ñ „ñð ññ °ð ÐL]ð ñ ð ñ „ñð ññ ¨4ð ÐFWð ñ ð ñ „ñð ñ
ñ 
¨ð 
ÐCTð 
ñ 
ð 
ñ „ñ
ð ññ  tð ÐBSð ñ ð ñ „ñð ñ
ñ 
 Tð 
Ð@Qð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
 Tð 
Ð@Qð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
¨ð 
ÐCTð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
 tð 
ÐBSð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
 Tð 
Ð@Qð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
 Tð 
Ð@Qð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
¨ð 
ÐCTð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
 tð 
ÐBSð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
 Tð 
Ð@Qð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
 Tð 
Ð@Qð 
ñ 
ð 
ñ „ñ
ð ññ ˜ð Ð4Eð ñ ð ñ „ñð ññ ˜$ð Ð5Fð ñ ð ñ „ñð ññ ˜$ð Ð5Fð ñ ð ñ „ñð ñ	ñ 	˜$ð 	Ð5Fð 	ñ 	ð 	ñ „ñ	ð ñ
ñ 
 tð 
ÐBSð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
¨ð 
ÐCTð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
¨ð 
ÐCTð 
ñ 
ð 
ñ „ñ
ð ñ
ñ 
¨ð 
ÐCTð 
ñ 
ð 
ñ „ñ
ð Ð ˜Hð 'Ù Ð"Eð'á!Ð#Gð'ñ "Ð#Gð'ñ !Ð"Eð	'ñ
 !Ð"Eð'ñ  Ð!Cð'ñ "Ð#Eð'ñ "Ð#Gð'ñ !Ð"Cð'ñ Ñ :ð'ñ Ñ :ð'ñ !Ð">ð'ñ  Ð!<ð'ñ  Ð!<ð'ñ Ð :ð'ñ  &Ð'Hð!'ñ" (Ñ)Lð#'ñ$ +OÙ)LÙ*Nñ)'ð 'ð 'ñ ô ð ð ð s   ¶= ½AÁ
A