o
     µÆgQ8 ã                   @   s8  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZmZmZmZmZmZ d dlmZ d dlZd dlmZ d dlmZmZ d dlmZ d d	lmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7 d d
l8m9Z9m:Z:m;Z;m<Z< d dl=m>Z> d dl?m@Z@ d dlAmBZB d dlCmDZD d dlEmFZF d dlGZGeG H¡ ZIeGjH J¡  eI Kd¡ZLeI Kd¡ZMeI Kd¡ZNdd„ ZOdd„ ZPdd„ ZQdd„ ZRdd„ ZSdd„ ZTdd „ ZUd!d"„ ZVd#d$„ ZWd%d&„ ZXd'd(„ ZYd)d*„ ZZd+d,„ Z[d-d.„ Z\d/Z]d0e]d1fd2d3„Z^d4d5„ Z_dS )6é    N)ÚBeautifulSoup)Úsettings)ÚONHNewsÚONHNewsDataÚONHNewsImageÚOneNewHomesDataÚOneNewHomesDevelopmentsDataÚOneNewHomesFilterDataÚOneNewsHomesDeveloper)Ú	ImageFile)Úslugify)Ú	DeveloperÚDeveloperTranslate)ÚContentFile)Ú
AttachmentÚAutoScoreRankÚBuildingTypeÚCurrencyÚDevelopmentÚDevelopmentOfferÚDevelopmentOfferTranslateÚDevelopmentPublicFacilityÚDevelopmentTranslateÚFaqCategoryÚFaqCategoryTranslateÚFeatureÚFeatureTranslateÚFlatÚFlatPriceHistoryÚ
KeyFeatureÚONHAreaÚONHDistrictÚONHPostcodeÚ
ONHStationÚONHZoneÚPaymentOptionÚPaymentOptionTranslateÚPaymentPlanÚPaymentPlanTranslateÚRankElementÚRankElementTranslate)ÚCityÚCountryÚLocationÚPublicFacility)ÚLanguage)ÚGEOSGeometry)ÚQ)Úurlparse)ÚOpenAIÚ
openai_keyÚopenai_pr_idÚopenai_org_idc               
   C   sd  d} d}d}t jddd}tjjdd ¡ }tjjd	d ¡ }|jd
krtj 	¡  
¡  tj 	¡  
¡  |j}|d }|j}tjjdt |¡d||d t|dƒ}	|	jddd„ d}
|
j dd¡ dd¡ dd¡ dd¡ dd¡}t d|tj¡}d}|r‹| d¡}t |¡}| d¡}	 | |kr‘n‰d| › d|› }d d!|› d"|› d#œ}zMt jd$|||dd%}| ¡ }| d&¡rð| d'¡}|t|d( ƒk rÍt|d( ƒ}W q‹d }| dkrØ| d)¡}tjj| d*t |d+ ¡||d, |d- |d( d. W n# ty } ztd/tj  ¡  !d0¡› d1|› ƒ W Y d }~nd }~ww | d7 } qŒd}d}d2}t jd3dd}tjjd4d ¡ }tjjd5d ¡ }|jd
kr0|j}|d }|j}tjjdt |¡d6||d t|dƒ}	|	jdd7d„ d}
|
j dd¡ dd¡ dd¡ dd¡ dd¡}t d|tj¡}d}|r˜| d¡}t |¡}| d¡}	 ||kr d S d|› d|› }d d8|› d9d#œ}zQt jd$|||dd%}| ¡ }| d&¡r| d'¡}|t|d( ƒk rÝt|d( ƒ}W q˜d }|dkré| d)¡}tjj| d:t |d+ ¡||d, |d- |d( d. W n# ty% } ztd/tj  ¡  !d0¡› d1|› ƒ W Y d }~nd }~ww | d7 } |d7 }q™d S );Né   é   zBhttps://1newhomes.com/assets/components/msearch2/action-search.phpz https://1newhomes.com/new-homes/é   ©ÚtimeoutÚlondon©Úslugúunited-kingdoméÈ   Ú	PHPSESSIDúhttps://1newhomes.com/)ÚidÚdataÚbase_urlÚcityÚcountryúhtml.parserÚscriptc                 S   ó   | od| v S ©NÚ
mse2Config© ©ÚtrM   rM   ú2/var/www/Entralon-Web-Backend/data_scraper/cron.pyÚ<lambda>.   ó    zget_data_1nh.<locals>.<lambda>©Útextú,
ú, ú,ú	Ú ú  Ú zmse2Config\s*=\s*({.*})Ú(30e35bbdbc0d5a572a05f1b709551ada3d2b5e7bÚkeyTzpage=z&action=filter&pageId=2923&key=z0application/x-www-form-urlencoded; charset=UTF-8a	  linkchain=/ ### /first-time-buyers ### /new-homes/ ### / ### /first-time-buyers ### /team/ ### /new-homes/ ### /new-homes/ ### /park-modern-london ### /park-modern-london ### /park-modern-london ### / ### /park-modern-london ### / ### /goodluck-hope-london ### / ### /goodluck-hope-london ### / ### /new-homes/ ### /developers ### /bellway-london/ ### /flats/ ### /harcourt-house-london ### /new-homes/ ### /goodluck-hope-london ### /hayes-village-london ### / ### /new-homes/ ### /the-beckett-london ### /the-beckett-london/gallery ### / ### /the-silk-district-london ### /the-silk-district-london ### /the-silk-district-london ### /the-silk-district-london ### /the-silk-district-london ### /the-silk-district-london ### /hepworth-place-london ### /hepworth-place-london ### /hepworth-place-london ### /hepworth-place-london ### /countryside-properties-london/ ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /developers ### /barratt-london/ ### /barratt-london/ ### / ### /1a-st-johns-wood-park-london ### /1a-st-johns-wood-park-london/gallery ### / ### /goodluck-hope-london ### / ### / ### / ### / ### / ### /developers ### / ### /first-time-buyers ### / ### /the-silk-district-london ### / ### /new-homes/ ### / ### / ### /new-homes/page-3 ### /new-homes/page-2 ### /new-homes/page-2 ### /new-homes/page-2 ### /new-homes/ ### /developers ### /taylor-wimpey-london/ ### /new-homes/ ### /new-homes/ ### /greenwich-millennium-village-london; _gcl_au=1.1.594264799.1719929307; _ga_JCEVRN83E0=GS1.1.1723452014.58.0.1723452016.58.0.0; _ga=GA1.1.65359444.1719929309; _ym_uid=1719929310452810339; _ym_d=1719929310; _fbp=fb.1.1722030370979.323930974364926; PHPSESSID=z<; cookie_agree=1; utm=; _ym_isad=1; _ym_visorc=w; PHPSESSID=)zContent-TypeÚCookieÚPOST)ÚheadersrD   r;   ÚsuccessrD   ÚpagesÚresultsJSONúhttps://1newhomes.comÚresultsÚtotalÚpage)rC   rE   rD   Ú	json_datarf   rg   rb   ú	error on ú%Y/%m/%d %H:%M:%Sú : zAhttps://1newhomes.ae/assets/components/msearch2/action-search.phpzhttps://1newhomes.ae/new-homes/Údubaiúunited-arab-emiratesúhttps://1newhomes.ae/c                 S   rJ   rK   rM   rN   rM   rM   rP   rQ   Z   rR   aö  linkchain=/ ### /news/a-15-quarter-climb-in-dubai-real-estate-prices ### /developers ### /emaar-properties-dubai/ ### / ### / ### / ### /new-homes/ ### /the-address-jumeirah-resort-and-spa-dubai ### /emaar-properties-dubai/ ### /the-address-jumeirah-resort-and-spa-dubai ### / ### /new-homes/ ### /prescott-real-estate-dubai/ ### /developers ### /new-homes/; _ga_SVS4S7PFEJ=GS1.1.1726536471.6.1.1726538323.0.0.0; _ga=GA1.1.415606180.1724083972; _ym_uid=1719929312736223957; _ym_d=1724083975; PHPSESSID=z*; cookie_agree=1; _ym_isad=1; _ym_visorc=wúhttps://1newhomes.ae)"ÚrequestsÚgetr+   ÚobjectsÚfilterÚfirstr,   Ústatus_coder   ÚallÚdeleter	   ÚcookiesrT   ÚcreateÚhtmlÚunescaper   ÚfindÚstringÚreplaceÚreÚsearchÚDOTALLÚgroupÚjsonÚloadsÚrequestÚintÚ	ExceptionÚprintÚdatetimeÚnowÚstrftime)ÚpÚapÚurlÚreq_corF   rG   ÚcorA   Úhtml_contentÚsoupÚ
script_tagÚscript_contentÚmse2Config_matchr]   rL   Úmse2Config_jsonÚpayloadr`   ÚresponseÚres_jsonÚresrh   ÚeÚp1rM   rM   rP   Úget_data_1nh   sÀ   
.


þ


0€,€ÿé
.




þ


0€,€ÿèòr   c                  C   sš   t jjddd} d}| D ]>}|j}d}|D ]-}z&| d¡}| d¡}| d¡}tjj|d\}	}
|r5||	_||	_|	 	¡  W q   Y qd|_
| 	¡  qd S )	NF)Újson_data_checkedÚjson_data__isnullrd   rŽ   ÚyearÚcoordr=   T)r   rr   rs   rh   rq   r   Úget_or_creater    ÚcoordinatesÚsaverž   )ÚdevelopmentsrE   ÚdevelopmentrD   ÚerrorÚdrŽ   r    r¡   ÚodÚ_rM   rM   rP   Úparse_json_development_data_1nh|   s(   



ðr«   c                  C   s  t   ¡ d } tj tddt| dB ¡d d… }d}d}d }d }|D ]W}|jd ur-|j}d|v rDtjjdd	 ¡ }tjjd
d	 ¡ }ntjjdd	 ¡ }tjjdd	 ¡ }t	|j
dƒ}|jddid}|t|ƒ7 }d}	|D ]û}
zÐ|
 d¡j}|
 d¡d }| d¡d }d }z||
 d¡ }tj|dd}|jdkr§tt |j¡dd}W n& tyÎ } zd}	tdtj ¡  d¡› d|› d|ƒ W Y d }~nd }~ww d }d }d }d }d }z|
jdd idd }| d¡d }|d!krö| d¡d" }W n" ty } z|
jdd idj}t|dd#}W Y d }~nd }~ww |d ur|d!krd }| d¡r:tjj|d d$… d	 ¡ }ntjj|› d%d	 ¡ }| d¡rZtjj|d d&… d	 ¡ }n|d u rktjj|› d'd	 ¡ }| d(¡rtjj|d d)… d	 ¡ }n|d u rtjj|› d*d	 ¡ }| d+¡r¤tjj|d d&… d	 ¡ }n|d u rµtjj|› d,d	 ¡ }|d u rÃtjj|d	 ¡ }|d u rÏtjj|d	}|j d u sÛ|j d!krù|d uræ| !¡ |_ n	| "d-d.¡ !¡ |_ ||_#||_$| %¡  t&jj'd/d0d1\}}t(jj'||d2\}}||_ ||_)| %¡  |d u r2td3tj ¡  d¡› d|› d4|› d5|› ƒ z#| d6¡rD|
jd7dd8idj}n| d9¡rT|
jd7dd:idj}W n   d }Y z½t*jj'|d	\}}||_ ||_+||_|d urw||_,||_-||_.||_/||_0d|_1| %¡  ztj|d;d}|jdkr¦t2 3|j¡|_4d|_1| %¡  W n% tyÌ } ztdtj ¡  d¡› d|› d<|ƒ W Y d }~nd }~ww z%tj|› d=d;d}|jdkrñt2 3|j¡|_5d|_6t   ¡ |_7| %¡  W n% ty } ztdtj ¡  d¡› d|› d>|ƒ W Y d }~nd }~ww W n' ty@ } zd}	tdtj ¡  d¡› d|› d?|ƒ W Y d }~nd }~ww W qn tyj } zd}	tdtj ¡  d¡› d@|j8› dA|ƒ W Y d }~qnd }~ww |	szd|_9t   ¡ |_:| %¡  q"tdBtj ¡  d¡› d|› dCƒ d S )DNé€Q F©Ú
is_checked©Úlast_check__lté
   rd   r   r<   r=   r?   rl   rm   rH   Úclasszsentence-v3-catalog©ÚattrsÚh2ÚaÚhrefú/éÿÿÿÿzdata-imgr:   r@   úfoo.png©ÚnameTri   rj   rk   ú image save errorÚbuilderrY   éþÿÿÿ©Úallow_unicodeéùÿÿÿú-londonéúÿÿÿú-dubaiÚ
developerséõÿÿÿú-developersr‚   ú-groupÚ-r[   ÚenglishÚen©r¼   Úcode©Ú	developerÚlanguagezon z  builder not Found builder name=z
 and slug=z.comÚspanzbuilding-addrz.aeÚdistrictr9   z error in download page: ú/galleryz error in download gallery: z error in save to data base: z : in development list [z]: zin z developments checked);Útimer   rr   rs   r1   rE   r+   rt   r,   r   rD   Úfind_allÚlenr|   rT   Úsplitrq   rp   ru   r   ÚioÚBytesIOÚcontentr‡   rˆ   r‰   rŠ   r‹   r   Úendswithr   ry   r¼   Útitler~   rF   rG   r¤   r/   r¢   r   r>   r   rŽ   Úimager¾   Úbuilder_slugÚbuilder_nameÚaddressÚdata_downloadedrz   r{   Úurl_dataÚgallery_dataÚgallery_downloadedÚgallery_last_downloadrC   r®   Ú
last_check)Úlcr¥   rE   ÚlrF   rG   r¦   r’   Údevelopment_listr§   Údlr¼   rŽ   r>   Úf1rÞ   Úimage_file_requestr›   Úf2Ú	builder_nÚdev_slugr¾   Ú	builder_srÑ   rª   Údev_trrá   r©   ÚreqrM   rM   rP   Úparse_development_data_1nh‘   s  $


€0€þ€€þ





,€€@€ 
€@€ €0€þ€2€þ
€&rô   c                  C   s¬   t jjddd d… } | D ]F}z|j}tj|dd}|jdkr-t |j	¡|_
d|_| ¡  W q tyS } ztdtj ¡  d	¡› d
|j› d|ƒ W Y d }~qd }~ww d S )NF)râ   r±   r9   r:   r@   Tri   rj   z :  development ú:)r   rr   rs   rŽ   rp   rq   ru   rz   r{   rT   rã   râ   r¤   r‡   rˆ   r‰   rŠ   r‹   r¼   )r¥   r¦   rŽ   ró   r›   rM   rM   rP   Údownload_development_data_1nh  s   
€2€ÿørö   c                  C   sÜ   t   ¡ d } tj tddt| dB ¡d d… }|D ]Q}z*|j› d}tj|dd}|jd	krEt	 
|j¡|_d
|_t   ¡ |_d|_| ¡  W q tyk } ztdtj ¡  d¡› d|j› d|ƒ W Y d }~qd }~ww d S )Ni / F)rå   )Úgallery_last_download__ltr±   rÔ   r9   r:   r@   Tri   rj   z) :  download gallery page of development rõ   )rÕ   r   rr   rs   r1   rŽ   rp   rq   ru   rz   r{   rT   rä   rå   ræ   Úgallery_checkr¤   r‡   rˆ   r‰   rŠ   r‹   r¼   )Últr¥   r¦   rŽ   ró   r›   rM   rM   rP   Ú download_development_gallery_1nh!  s$   $

€2€ÿörú   c                 C   sÂ  | D ]Ü}z¶|  d¡}d|v r4z| dd¡ d¡ d¡}W n   z| dd¡ d¡ d¡}W n   Y Y t|d d… d	d
}z	t|  d¡ƒ}W n   d }Y z|  d¡}W n   d }Y |  d¡  d¡}td|  d¡› d|  d¡› dƒ}	tjj	|	d\}
}|d d… |
_
||
_|
 ¡  tjj	|	|
|d\}}|d d… |_
||_| ¡  tjj	||d\}}||_||_| ¡  W q tyÞ } ztdtj ¡  d¡› d|› d|ƒ W Y d }~qd }~ww d S )Nr¼   z\\uú\\ú\Úasciizunicode-escapeÚutf8é–   TrÀ   ÚdistanceÚdurationÚgeometryÚlocationúPOINT(Úlngr[   Úlatú)©r£   )r£   r  Útype_id©r¦   Úpublic_facilityri   rj   z :  create public facility rõ   )rq   r~   ÚencodeÚdecoder   Úfloatr0   r-   rr   r¢   r¼   r>   r¤   r.   r   r   r  r‡   rˆ   r‰   rŠ   r‹   )ÚobjsÚtyper¦   ÚobjÚobj_nameÚobj_slugÚobj_distanceÚobj_durationÚobj_locationÚpntÚnew_locationrª   ÚpfÚdpfr›   rM   rM   rP   Úcreate_public_facility2  sD   
€"0€ÿãr  c            “      C   sâ  t jjddd\} }tjjdd ¡ dkrtj ¡ jdd t ¡ d }tjjt	dd	t	|d
B dddd d… }|D ]}d|_
d|_| ¡  q;|D ]£}d}d }|j}|j}	t|	dd}
z;| d¡rztjjdd}tjjdd}tjjdd}n!| d¡r•tjjdd}tjjdd}tjjdd}nd }d }d }d }zt|j ¡ ƒ}|jj|_W n   d }Y zt|j ¡ ƒ}|jj|_W n   d }Y z#z
|j}t|dƒ}W n   tj|dd}t |j¡}t|dƒ}Y W n" ty } zt d|› ƒ d|_!| ¡  W Y d }~W qJd }~ww d }z|j"ddidj"ddid}d  #d!d"„ |j$D ƒ¡}W n   Y z&| %d#d$¡ %d%d ¡ %d&d ¡ %d'd ¡ %d(d ¡ %d)d ¡ %d*d+¡ %d,d-¡ &¡ }W n   Y d }z|j"dd.idj"ddid}d  #d/d"„ |j$D ƒ¡}W n   Y z&| %d#d$¡ %d%d ¡ %d&d ¡ %d'd ¡ %d(d ¡ %d)d ¡ %d*d+¡ %d,d-¡ &¡ }W n   Y d }z|j"dd0id}| "d1¡ '¡  W n   Y z1d  #d2d"„ |j$D ƒ¡}| %d#d$¡ %d%d ¡ %d&d ¡ %d'd ¡ %d(d ¡ %d)d ¡ %d*d+¡ %d,d-¡ &¡ }W n   Y d }d3}d3}zE|j(}|d u r|j"dd4idj"dd5idj}n|d6krd7}| %d%d ¡ %d&d ¡ %d'd ¡ %d(d ¡ %d)d ¡ %d*d+¡ %d,d-¡ %d8d9¡ &¡ }W n   d }Y |d u sQ|d krTd}n| )¡ d:v }t*j* +¡ }d }|sô| )¡ }g } d;|v ru| ,d;¡} nd<|v r€| ,d<¡} nd9|v r‹| ,d9¡} n|  -|¡ | D ]a}!d=}"d}#d>|!v rÓ|! ,d>d?¡}$|$d3 }%|$d? }&|%d@krµdA}"dB}#n$|%dCkr¿dD}"d}#n|%dEkrÉdF}"d}#n|%dGkrÒd=}"dB}#ndB}#d=}"|!}&t* .t/|&ƒ|"|#¡}'|d u rë|'}q’|'|krò|'}q’|d ur|j(|j(k rd}z|j"ddHidj"dd5idj0ddIid}(W n   g }(Y dJgdKgdLgdMgdJdKgdKdMgdJdLgg dN¢dKdLgg dO¢dJdMgg dP¢g dN¢dQœ})z| 1dR¡d3 j0dd5id}*W n   g }*Y g }+|*D ]=}!|! d¡},t |!j¡}	|	 %d%d ¡ %d&d ¡ %d'd ¡ %d(d ¡ %d)d ¡ %d*d+¡ %d,d-¡ &¡ }	dS|,v r›|+ -|)dT |	dUœ¡ q_z|j"ddVidj"dd5idj}W n   Y |rî| )¡  2dW¡rÁdX}-nd?}-| %dYd ¡ %dWd ¡ ,d;¡}z
t3|d3 ƒ|- }W n   Y z
t3|dZ ƒ|- }W n   Y g }.z„|j"dd[id 0d\¡}/|j"dd[id 0d]¡}0|/rst4t5|/ƒƒD ]b}!z|0|! j )¡ d^kr"|0|! jnd_}1W n   d_}1Y zt3|/|!  d`¡ƒ}2W n   d3}2Y zt3|/|!  da¡ƒ}3W n   d3}3Y z|3|2 }4W n   |1d_kra|n|}4Y |. -|1|2|3|4|2d3kdbœ¡ qW n   Y d}5z|j"ddcidj"dddidj )¡ dev }5W n   z|j"ddfidj"dd5idj )¡ dev }5W n   Y Y |j0ddgid}6|j0ddhid}7|j0ddiid}8|j0ddjid}9|j0ddkid}:|j0ddlid};d }<d }=d }>d }?g }@|j"dmdndo„ dp}A|j"dmdqdo„ dp}B|Br¾|Bj6 %drds¡}Bt7 8dt|Bt7j9¡}C|Cr¾|C :d?¡}?t; <|?¡}?|?D ]¡}DzT|D du¡d3  )¡ d^kr.d_}EnC|D du¡d3  )¡ dvkr=dw}En4|D du¡d3  )¡ dxkrLdy}En%|D du¡d3  )¡ dzkr[d{}En|D du¡d3  )¡ d|krjd}}En|D du¡d3 }EW n   d_}EY z
t3|D d~d3¡ƒ}FW n   d3}FY z
t3|D dd3¡ƒ}GW n   d3}GY z
t3|D d€d3¡ƒ}HW n   d3}HY |. -|E|F|G|H|D d¡d kdbœ¡ qd }I|Arž|Aj6 %d‚dƒ¡ %d„dƒ¡ %d…d ¡ %d†d‡¡ %d†d‡¡ %dˆd+¡ %drds¡}Jt7 8d‰|Jt7j9¡}K|Krò|K :d?¡}<t7 8dŠ|Jt7j9¡}L|Lr|L :d?¡}>zt; <|>¡}>W n ty } zd }>W Y d }~nd }~ww t7 8d‹|Jt7j9¡}M|MrU|M :d?¡}@t7 =dŒd|@¡}@|@ %d+dŽ¡ %dd¡ %d‘d’¡ %d“d”¡ %d•d–¡ %dsd ¡}@t; <|@¡}@d—}Nd—}Oz@t7 8d˜|Jt7j9¡}P|Prj|P :d?¡}Nt7 8d™|Jt7j9¡}Q|Qrz|Q :d?¡}Ot>dš|N› d‡|O› d›ƒ}It?jj|Idœ\}=}R|Rr˜|	|=_|= ¡  W n   Y z¡z	t@jj|
d}SW n   t@jjA|
d}SY |SjBd u r½|	|S_B|SjCd u rÇ|jD|S_C|Sjd u rÕ|d urÕ||S_|Sjd u rã|d urã||S_|SjEd u sï|SjEd3krò||S_E|SjFd u sþ|SjFd3kr||S_F|=|S_G|SjF|SjE d |S_H|5|S_I|>|S_J||S_K|Ir|I|S_L|r"||S_M|r(||S_N|r.||S_O|r4||S_P|r:||S_Q|S ¡  W n. tym } z!t džt*j* +¡  RdŸ¡› d |j› d¡|ƒ d|_!| ¡  W Y d }~W qJd }~ww z
tSjj| |Sd¢}TW n   tSjjA| |Sd¢}TY |TjBd u rŽ|	|T_B|TjTd u r—|
|T_T|TjUd u r£|r£||T_U|TjVd u r¯|r¯||T_V|rµ||T_W|TjXd u r¾|<|T_X|T ¡  |(D ]F}Uz0|Uj )¡  %d£d ¡ B¡ }Vt|Vdd}Wz	tYjj|Wd}XW n   tYjjA|V|Wd¤}XY |SjZ [|X¡ W qÄ ty
 } zW Y d }~qÄd }~ww |.D ]©}Yz}z	t/|Y d¥¡ƒ}ZW n   d3}ZY t\jj|S|Zd¦\}[}\|\r3|Y d€¡|[_H|[jHd3kr?|Y d€¡|[_H|Y dW¡|[_]|Y d§¡|[_^|Y d¡dk|[_I|[ ¡  t_jj|[d¨ `d©¡ a¡ }]|]d u sq|]jb|Y dW¡krŠ|Y dWd3¡d3krŠt_jjA|[|Y dW¡t ¡ dª d«}]W q ty¶ } zt džt*j* +¡  RdŸ¡› d |j› d¬|Y› d­|ƒ W Y d }~qd }~ww |@D ]o}^|^d®  )¡ d¯v rÇd_}_n6|^d®  )¡ d°v rÓdw}_n*|^d®  )¡ d±v rßdy}_n|^d®  )¡ d²v rëd{}_n|^d®  )¡ d³v r÷d}}_n|^d® d3 }_zt\jj|S|_d¦\}`}W n   Y q¹|^d´ }a|aD ]}t_jj|`|dµ |d¶ d« 	qq¹|6D ]j}bz=|b "dz¡j %d%d ¡ %d­d ¡ %d'd ¡ %d)d ¡ &¡ }ctcjj|S|cd·\}d}z
tdjj| |dd¸ W n   tdjjA| |d|cd¹ Y W 	q+ t	y• } zt džt*j* +¡  RdŸ¡› d |j› dº|bj› d­|ƒ W Y d }~	q+d }~ww |7D ]¥}ezx|ej"dd»idjd dZ… }fz
t3|f %d¼d½¡ƒ}fW n   d3}fY |ej"dd¾idj %d%d ¡ %d­d ¡ %d'd ¡ %d)d ¡ &¡ }gt|gdd}hz
tejj|S|hd¿}dW n   tejjA|S|gdÀ}dY |f|d_f|d ¡  z
tgjj| |ddÁ W n   tgjjA| |d|gdÂ Y W 	q˜ t
y= } zt džt*j* +¡  RdŸ¡› d |j› dÃ|ej› d­|ƒ W Y d }~	q˜d }~ww |8D ]{}izN|ij %d%d ¡ %d­d ¡ %d'd ¡ %d)d ¡ &¡ }ct|cƒ}jthjj|c|jd¤\}d}dÄ|i dg ¡v 
ru|Sji [|d¡ z
tjjj| |ddÅ W n   tjjjA| |d|c|jdÆ Y W 
q@ t
y» } zt džt*j* +¡  RdŸ¡› d |j› dÇ|ij› d­|ƒ W Y d }~
q@d }~ww |9D ]­}kz|k "dÈ¡ k¡ }l|kj"ddÉidj %d%d ¡ %d­d ¡ %d'd ¡ %d)d ¡ &¡ }mt|mdd}nz|kj"ddÊidj}oW n   d }oY z|o %d%d ¡ %d­d ¡ %d'd ¡ %d)d ¡ &¡ }oW n   Y dË|m )¡ v rVz|o %d‡d ¡}p|p|S_l|p ,dÌ¡}q|qd3 |S_m|qdZ |S_n|S ¡  W n tyU } zt dÍ|ƒ W Y d }~nd }~ww dÎ|m )¡ v r”z|o %d‡d ¡}r|r|S_o|r ,dÌ¡}s|sd3 |S_p|sdZ |S_q|S ¡  W n ty“ } zt dÏ|ƒ W Y d }~nd }~ww dÐ|m )¡ v râz.|o )¡  %d£d ¡ B¡ }Vt|Vdd}Wz	tYjj|Wd}XW n   tYjjA|V|Wd¤}XY |SjZ [|X¡ W n tyá } zt dÑ|ƒ W Y d }~nd }~ww zt3|oƒ}tW n   d }tY z	trjj|nd}uW n   trjjA|n|mdÒ}uY |ujsd u r|l|u_s|u ¡  z
ttjj| |udÓ W n   ttjjA| |u|m|ndÔ Y tujj|S|udÕ\}v}|o|v_v|t|v_w|v ¡  W 
q¾ tyl } zt džt*j* +¡  RdŸ¡› d |j› dÖ|kjx› d­|ƒ W Y d }~
q¾d }~ww d3}w|;D ]~}xzU|x "d×¡j %d%d ¡ %d­d ¡ %d'd ¡ %d)d ¡ &¡ }yztyjj|ydØ\}z}W n   tyjjA|ydØ}zY |w|z_z|z ¡  |wd?7 }wz
t{jj| |zdÙ W n   t{jjA| |z|ydÚ Y W qq tyï } zt džt*j* +¡  RdŸ¡› d |j› dÛ|ƒ W Y d }~qqd }~ww |:D ]á}{z´|{ "dÈ¡ k¡  %d%d ¡ %d­d ¡ %d'd ¡ %d)d ¡ &¡ }||{j"ddÜidj %d%d ¡ %d­d ¡ %d'd ¡ %d)d ¡ &¡ }}t|}dd}~d }z|{j"ddÝid dÞ¡}W n tyU } z
t |ƒ W Y d }~nd }~ww z	t|jj|~d}€W n   t|jjA|~|}dÒ}€Y |€jsd u ru|||€_s|€ ¡  z
t}jj| |€dß W n   t}jjA| |€|}|~dà Y t~jj|S|€dá\}}|r¢||_| ¡  W qò tyÓ } zt džt*j* +¡  RdŸ¡› d |j› dâ|kj› d­|ƒ W Y d }~qòd }~ww |+D ]C}‚z;t|‚d® d dã… dd}ƒt€jj|ƒ|‚dä då\}„}|‚d® |„_d?|„_d|„_|„ ¡  t‚jj|S|„dæ\}…}d?|…_|… ¡  W qÖ   Y qÖz|> dçi ¡ dè¡}†tƒ|†dé|Sƒ W n   Y z|> dêi ¡ dè¡}‡tƒ|‡dë|Sƒ W n   Y z|> dìi ¡ dè¡}ˆtƒ|ˆd?|Sƒ W n   Y z|> díi ¡ dè¡}‰tƒ|‰dî|Sƒ W n   Y z|> dïi ¡ dè¡}Štƒ|Šdð|Sƒ W n   Y z|> dñi ¡ dè¡}‹tƒ|‹dò|Sƒ W n   Y z|> dói ¡ dè¡}Œtƒ|Œdô|Sƒ W n   Y z|> dõi ¡ dè¡}tƒ|dA|Sƒ W n   Y z|> döi ¡ dè¡}Žtƒ|Žd|Sƒ W n   Y t„|Sj… ¡  †d÷dødùd€dúdû¡ƒ}z|SjCjdü|SjCjT› |SjCj› |SjCj‡dýœ}W n   d }Y z|SjMj|SjMjˆ|SjMj‰dþœ}‘W n   d }‘Y dÿ|Sj‡i d |Sj‡“dÉ|SjB“dd|SjT› “dû|Sj^“d|SjX“d€|SjH“d|SjE“d|SjF“d|Sjl“d|Sjo“d|SjŠ“d	|SjP“d
|SjK“dù|SjI“d|Sj› “dÎ|“d|“d|‘i¥d|SjLj‹|SjLjŒgdœdœ}’|’|S_t ¡ |S_Ž|S ¡  W n+ tyÄ } zd}|}t džt*j* +¡  RdŸ¡› d |j› d|ƒ W Y d }~nd }~ww |sØd|_t ¡ |_d|_
| ¡  qJd|_t ¡ |_d|_|› |_‘d|_
| ¡  qJd S (  NrË   rÌ   rÍ   T)Úin_checkéZ   Fr¬   r­   r¯   )r  Úname__isnullé(   rÀ   rB   ÚGBP)rÎ   r<   r=   r?   rn   ÚAEDrl   rm   rH   r9   r:   zerror on read html: r²   ztab-descriptionr³   zcollapsed-textrY   c                 S   ó   g | ]}|› ‘qS rM   rM   ©Ú.0ÚitemrM   rM   rP   Ú
<listcomp>  ó    z&create_development.<locals>.<listcomp>Ú	1newhomesÚEntralonú\nÚ
ú\rúú\tú\'ú'ú\/r¸   ztab-featuresc                 S   r"  rM   rM   r#  rM   rM   rP   r&  —  r'  zneighborhood-contentr¶   c                 S   r"  rM   rM   r#  rM   rM   rP   r&  ¢  r'  r   Ú
completionÚvalueÚfinishedzReady to movez \xe2\x80\x93 z - )zready to moveÚreadytomoveu    â€“ z xe2x80x93 é   zq/r7   Ú1é   é   Ú2é   Ú3é	   Ú4zproperty-typeÚstationúsubway-stationsúrail-stationsútrain-stationsúdlr-stations)r@  rA  rC  )rA  rB  rC  )r@  rA  rB  )ÚuÚrÚor¨   ÚurÚrdÚuoÚurdÚroÚrodÚudÚurorJ  z.prop.stationz	station-urD  )r  r¼   ÚsquareÚsqmg%½^~È·?Úsqftr¹   zf-countÚinputÚlabelÚsÚ0zdata-sqmzdata-all)ÚnumrP  rv   ÚareaÚsoldoutzprice-block--valueÚbutton)rX  zsold outzprice-block--soldoutzproperty-offerzpayment-plan--itemzpayment-methodztth_table--columnzexpert--circle-wrapperzfaq-wrapperrI   c                 S   rJ   )NÚpage_configrM   rN   rM   rM   rP   rQ   ,  rR   z$create_development.<locals>.<lambda>rS   c                 S   rJ   )NÚpriceSelectorDatarM   rN   rM   rM   rP   rQ   -  rR   rü   rû   z&const priceSelectorData\s*=\s*(\[.*\])ÚroomsÚvÚ10000rO   Ú10001rŒ   Ú10002r¨   Ú10003ÚpriceM2Ú
priceWholerW  rX  rU   rV   rW   rX   rZ   r[   z\"z"address"\s*:\s*"(.*?)",z'const infrastructurePlaces\s*=\s*({.*})z5const historyPricesData\s*=\s*(\[.*\])\\\\n\\\\n     znew Date\((\d+)\)z\1ú"zname:z"name":zdata:z"data":zx:z"x":zy:z"y":g        z"coordinates_long"\s*:\s*(.*?),z"coordinates_lat"\s*:\s*(.*?),r  r  r  r8   ri   rj   ú development z create new development:)rÑ   r¦   z\\n\s+)r¼   r>   rV  )r¦   Úbedrooms_numrv   )ÚflatrÕ   iè  )rg  ÚpricerÕ   z create new flat by data rõ   r¼   )ÚstudiorY   )Úvilla)Ú	townhouse)Ú	penthouse)ÚduplexrD   ÚyÚx)r¦   rT   )rÑ   Údevelopment_offer)rÑ   rp  rT   z create new offer zpayment-plan--valueú,Ú.zpayment-plan--title)r¦   r>   )r¦   rÝ   )rÑ   Úpayment_plan)rÑ   rs  rÝ   z create new payment plan Úactive)rÑ   Úpayment_option)rÑ   ru  r¼   r>   z create new payment method ÚsvgrÝ   ztth-textÚfloorsrÊ   zadd floor: Úflatsz
add flat: zproperty typezadd property type: )r>   r¼   )rÑ   Úfeature)rÑ   ry  r¼   r>   )r¦   ry  z create new key feature Úh3)rÝ   )rÑ   Úcategory)rÑ   r{  rÝ   z create new faqs group:zexpert--circle-titlezexpert--circleú
data-value)rÑ   Úrank_element)rÑ   r}  r¼   r>   )r¦   Úelementz create new auto score rank rÿ   r  )r>   r	  r
  Úparkre   é   Úcafeé   Úprimary_schoolÚbus_stationé   Úgymé   Úhospitalé   Úsupermarketé   Útrain_stationÚsubway_stationÚbathrooms_numrf  Úis_sold_outÚprice_per_meterÚ
base_pricez
developer-)r¼   r>   rÞ   rC   )r¼   rÎ   Úsymbolr   rC   r>   z	property-rá   Úmin_areaÚmax_areaÚfloorrg  Úis_featuredÚcompleted_dateÚis_ready_to_moverÞ   rÐ   ÚcurrencyÚPoint)r  r£   )r  rC   Ú
propertiesr  z create:)’r/   rr   r¢   r   rs   Úcountrv   ÚupdaterÕ   r1   r  r®   r¤   rŽ   r¼   r   Ú
startswithr   rq   r+   r,   r   rÞ   ÚreadÚimage_mobilerã   r   rp   rz   r{   rT   r‡   rˆ   Úgallery_in_checkr|   ÚjoinÚcontentsr~   ÚstripÚ	decomposer    Úlowerr‰   rŠ   rØ   ÚappendÚdater†   rÖ   ÚselectrÜ   r  Úranger×   r}   r   r€   r   r‚   rƒ   r„   Úsubr0   r-   r   ry   rÝ   rÐ   r¾   r“  r”  r  rW  r  Úplacesr˜  r£   Údefault_currencyrF   rG   r—  Úcompleted_atr‹   r   r>   ÚdescriptionÚfeatures_descriptionÚneighborhoodrá   r   Úbuilding_typeÚaddr   r  r‘  r   Úorder_byÚlastrh  r   r   r'   Úpercentr(   r%   ru  r&   Úprettifyr•  Ú	min_floorÚ	max_floorrg  Úmin_flatÚmax_flatr   Ú	icon_textr   r   r3  rV  Úkf_titler   Úorderr   r)   r*   r   Úrankr.   Ú	is_activer   r  Úlistrx  ÚvaluesrC   rÎ   r’  r–  ro  rn  Úgeo_jsonÚlast_createrç   Ú
have_errorr§   )“rÑ   rª   rè   r¥   r¨   r¦   r§   ÚerrrŽ   r¼   r>   r­  rF   rG   Únew_fileÚ	new_file1Ú	html_coder’   ró   r›   r¯  ÚdesÚf_descriptionr±  Ú
area_ranger“  r”  r—  r˜  rŠ   r®  ÚcdÚarÚiÚmonthÚdayÚadÚqr    ÚcaÚp_tsÚstation_typesÚstation_divÚstation_topÚi_crŒ   Úbedrooms_rangeÚbedrooms_inputÚbedrooms_labelÚdata_numÚdata_sqmÚdata_allÚ	data_arear  Úproperty_offersÚpayment_plansÚpayment_methodsÚkey_featuresÚauto_scoresÚfaqs_groupsrá   r  r¬  ÚpricesÚprices_historyr“   Úscript_priceÚpriceSelectorData_matchrh  Úbdr_num_dataÚbdr_sqm_dataÚbdr_all_dataÚbdr_area_datar£   r”   Úaddress_matchÚinfrastructurePlaces_matchÚhistoryPricesData_matchr  r  Úcoordinates_long_matchÚcoordinates_lat_matchry   Únew_developmentÚdevelopment_translateÚptÚpt_nameÚpt_slugÚproperty_typeÚbedroomÚbedroom_numÚnew_flatÚcreate_flatÚfphÚphÚbnrg  Úph_dataÚofferÚpo_nameÚdors  Úplan_percentÚ
plan_titleÚ	plan_slugÚpayment_methodÚpo_slugÚkey_featureÚkf_iconr½  Úkf_slugÚkf_valuer•  rw  Úp_flatÚp_flatsÚkf_numÚnew_fÚnew_kfÚfg_orderÚ
faqs_groupÚfaqs_group_titler{  Ú
auto_scoreÚas_iconÚas_titleÚas_slugÚas_valueÚnew_reÚnew_asrÚstÚst_slugr  r  ÚparksÚcafesÚprimary_schoolsÚbus_stationsÚgymsÚ	hospitalsÚsupermarketsÚtrain_stationsÚsubway_stationsrx  rÐ   r™  rÃ  rM   rM   rP   Úcreate_developmentS  sæ  *



€€üLLL*L









€€.ó 
@
€"(û€*,€

€û>

€ÿ
4


€&€ü€ . €:€ÿÿ0€<€ÿ6€<€ÿ*€<€ÿ
6.


&€ 


&€ &€ <€ÿ0€4€ÿ26$€ <€ÿ
,ÿþýüûúùø	÷
öõôóò
ñðïîþþé
4€ý



    œr(  c                  C   s‚  d} t jjdd ¡ dkrt j ¡ jdd t ¡ d d d d }t jjtdd	t|d
B dddd d… }|D ]}d|_d|_	| 
¡  q9|D ]õ}|jd urS|j} |jd u s]|jdkred|_| 
¡  qHz«|j}t|dƒ}|jddid}t|jdd}tjj|d}	|D ]‡}
d}zU|
 d¡}|r¯| ¡ dkržd}n| ¡ dkr§d}n| ¡ dkr¯d}|
 d¡}|D ])}| d¡}|d }| d|j¡}| d|j¡}d  | d!¡d"  d#¡d d"… ¡}zX| d$¡j}|r8d%| ¡ v rôd}nDd| ¡ v rýd}n;d| ¡ v rd}n1d&| ¡ v rd'}n'd(| ¡ v rd)}nd*| ¡ v r%d}nd+| ¡ v r/d,}n	d-| ¡ v r8d,}W n	   | d¡}Y d }zn| | }tj|d.d/}|› d | d!¡d" › }|jd0krmtt |j¡|d1}ztjj||	|d2 W n&   ztjj ||	||||d3 W n t!y } zW Y d }~nd }~ww Y d|_t ¡ |_"d|_	| 
¡  |	 
¡  W q¶ t!yà } z d|_	| 
¡  t#d4t$j$ %¡  &d5¡› d6|j› d7|ƒ W Y d }~q¶d }~ww W q‡ t!y } z d|_	| 
¡  t#d4t$j$ %¡  &d5¡› d8|j› d9|ƒ W Y d }~q‡d }~ww W qH t!y> } z d|_	| 
¡  t#d4t$j$ %¡  &d5¡› d:|j› d;|ƒ W Y d }~qHd }~ww d S )<Nrd   T)r¡  é2   Fr7   é   é<   )rø   )Úgallery_last_check__lt)rå   r¡  é   rY   rH   r²   zmedia-gallery--wrapperr³   rÀ   r=   r   rC   Úinteriorr8   z	site-plané   ÚexteriorÚfigureÚimgÚsrcrÝ   ÚaltrÊ   r¸   r¹   rr  Ú
figcaptionzprimary photoÚ	amenitiesr8  z
floor planr…  z	site planz	flat planr;  Úlayoutsr±   r:   r@   r»   )Ú	file_namer¦   r{  )r8  r¦   rÝ   r4  rÞ   r{  ri   rj   rk   r½   re  z attachment:z attachment for development ú not created:)'r   rr   rs   rœ  rv   r  rÕ   r1   rø   r¡  r¤   rE   rä   rå   r   rÖ   r   r¼   r   rq   r¦  r|   r¢  rØ   rT   rp   ru   r   rÙ   rÚ   rÛ   r   ry   r‡   Úgallery_last_checkrˆ   r‰   rŠ   r‹   )rE   rù   r¥   r¨   r¦   rD   r’   Údivsr>   ÚdevÚdivr{  Údiv_idÚfiguresr1  r2  r3  rÝ   r4  r8  r5  rì   rÞ   rí   Ú
image_namer›   rM   rM   rP   Úadd_attachment_to_developmentÀ  s²   *







"€€ €
2€ýÝ(2€ýÏ52€ý¾rA  c                  C   s–  t   ¡ d } tj tddt| dB ¡ ¡ }|rIzú|j}|j}|j}t	|dƒ}|j
dddid	j}t	|dƒ}|jd
ddid	}|D ]j}	|	j}
t|
dd}|	 d¡}z6t dd|¡}ztjj||||d}W n   tjj|
||||d}Y d|› |_d|› d|_| ¡  W qC ty­ } ztdtj ¡  d¡› d|
› d|ƒ W Y d }~qCd }~ww |jd
ddid	}|D ]A}|j}
t|
dd}| d¡}ztjj|
||||d}W q¹ tyú } ztdtj ¡  d¡› d|
› d|ƒ W Y d }~q¹d }~ww |jd
ddid	}|D ]D}|j}
t|
dd}| d¡}ztjj|
||||d}W q tyJ } ztdtj ¡  d¡› d|
› d|ƒ W Y d }~qd }~ww z|j
dd id	 
d!¡ d"¡}W n   g }Y |D ]J}| 
d#¡j}
t|
dd}| 
d$¡ d%¡}ztjj|
||||d}W qd ty® } ztdtj ¡  d¡› d&|
› d|ƒ W Y d }~qdd }~ww z|j
dd'id	 
d!¡ d"¡}W n   g }Y |D ]J}| 
d#¡j}
t|
dd}| 
d$¡ d%¡}ztjj|
||||d}W qÈ ty } ztdtj ¡  d¡› d(|
› d|ƒ W Y d }~qÈd }~ww W n& ty: } ztdtj ¡  d¡› d)|j › d|ƒ W Y d }~nd }~ww d|_!t   ¡ |_"| ¡  d S d S )*Nr¬   Fr­   r¯   rH   rI   rC   Úgeoselector_modal_templater³   rÒ   zdata-tvÚsubwayTrÀ   r|  z\drY   )r>   rÎ   rF   rG   )r¼   rÎ   r>   rF   rG   zstation-zicons/station-z.svgri   rj   z	 station r9  rÓ   z
 district Úpostcodez
 postcode r²   rW  ÚulÚlirS  rR  r3  z area Úzonez zone z filter )#rÕ   r	   rr   rs   r1   rt   rD   rF   rG   r   r|   rT   rÖ   r   rq   r   r«  r#   ry   r  Úiconr¤   r‡   rˆ   r‰   rŠ   r‹   r!   r¢   r"   r    r$   rC   r®   rç   )rè   rs   rÉ  rF   rG   rz   r“   r’   Ústationsr?  r¼   r>   rÎ   Ús_typer  r›   Ú	districtsrÓ   Ú	postcodesrD  ÚareasrW  ÚzonesrG  rM   rM   rP   Úparse_filter_data_1nh  s¦    


0€ÿ
0€ÿ
2€ÿ"2€ÿ"2€ÿú
2€ÿ
·rO  c                  C   sî  d} | › d}t j|dd}t |j¡}t|dƒ}|jddid}tjjd	d
}|D ]Ž}zk|j	j	j
}| › |d › }	|d dd… }
| › |jd › }d }zt j|dd}|jdkrdtt |j¡dd}W n ty| } ztd|ƒ W Y d }~nd }~ww tjj|
d
\}}||_|	|_||_||_| ¡  W q* ty¸ } ztdtj ¡  d¡› d|ƒ W Y d }~q*d }~ww d} | › d}t j|dd}t |j¡}t|dƒ}|jddid}tjjdd
}|D ]‘}zm|j	j	j
}| › |d › }	|d dd… }
| › |jd › }d }zt j|dd}|jdkrtt |j¡dd}W n ty7 } ztd|ƒ W Y d }~nd }~ww tjj|
d
\}}||_|	|_||_||_| ¡  W qã tyt } ztdtj ¡  d¡› d|ƒ W Y d }~qãd }~ww d S )Nrd   z/developersr9   r:   rH   r²   zdevelopers-list--itemr³   r<   r=   r·   r7   r¹   r3  r@   rº   r»   zimage save errorri   rj   z developer:ro   rl   )rp   rq   rz   r{   rT   r   rÖ   r+   rr   r=  r}   r2  ru   r   rÙ   rÚ   rÛ   r‡   rˆ   r
   r¢   r¼   rŽ   rF   rÞ   r¤   r‰   rŠ   r‹   )rE   rŽ   ró   Ú	html_pager’   Údevelopers_listrF   rë   r¼   Úlinkr>   rÞ   Úfrí   r›   r©   rª   rM   rM   rP   Úget_data_1nh_developera  s†   



€€ÿ*€ÿ


€€ÿ*€ÿírT  c                  C   sø  t jjddd d… } tjjddd\}}| D ]á}z¸|j}|j}|j}|j}t	|dd}t
|j ¡ ƒ}	|jj|	_tj|d	d
}
|
jdkrÒt |
j¡}t|dƒ}z"|jddidd }zt|ƒ}|j› d|j› }W n   d }Y W n   d }Y |jddid}z|jddid}| d¡ W n   Y z|jddid}| d¡ W n   Y z|jddid}| d¡ W n   Y |› }z&| dd¡ dd¡ dd¡ dd¡ dd¡ dd¡ dd¡ d d!¡ ¡ }W n   Y d }| d"¡rtjj|d d#… d$ ¡ }ntjj|› d%d$ ¡ }| d&¡r#tjj|d d'… d$ ¡ }n|d u r4tjj|› d(d$ ¡ }| d)¡rHtjj|d d*… d$ ¡ }n|d u rYtjj|› d+d$ ¡ }| d,¡rmtjj|d d'… d$ ¡ }n|d u r~tjj|› d-d$ ¡ }|d u rŒtjj|d$ ¡ }|d u r˜tjj|d$}||_|	|_||_||_||_|  ¡  t!jj||d.\}}||_||_"|j#sÂ||_#|  ¡  d|_$t% %¡ |_&|  ¡  W q t'yù } zt(d/t)j) *¡  +d0¡› d1|j› d2|ƒ W Y d }~qd }~ww d S )3NFr­   r)  rË   rÌ   rÍ   TrÀ   é   r:   r@   rH   r²   zbtn gar³   r·   z://ústicky-block--contentztitle-contentrY   zdeveloper-logozdeveloper-offersr(  r)  r*  r+  r,  r-  r.  r/  r0  r1  r¸   r<   rÂ   r=   rÃ   rl   rÄ   rÅ   rÆ   rÇ   rÈ   r‚   rÉ   rÏ   ri   rj   z developer rõ   ),r
   rr   rs   r/   r¢   r¼   rŽ   rF   rG   r   r   rÞ   rŸ  rp   rq   ru   rz   r{   rT   r   r|   r2   ÚschemeÚnetlocÚreplaceWithr~   r¤  rÜ   r   rt   ry   Úweb_addressr¤   r   r>   r¯  r®   rÕ   rç   r‡   rˆ   r‰   rŠ   r‹   )rÆ   rÑ   rª   rÐ   r¼   rR  rF   rG   r>   rÇ  ró   rP  r’   ÚwebsiteÚ
parsed_urlrÊ  Úh1r2  Úoffr¯  r<  rò   r›   rM   rM   rP   Úcreate_developerœ  s¢   


€L





€2€ÿ¶r_  c                  C   s>  dd l } dd l}tj}|› d}d}| ¡ }| j|ddD ]x\}}}|D ];}	| j ||	¡}
|| j |
¡ }||d kr_z|  |
¡ W q$ t	y^ } zt
d|
› d|› ƒ W Y d }~q$d }~ww q$|D ]2}	| j ||	¡}|  |¡s”z|  |¡ W qb t	y“ } zt
d	|› d|› ƒ W Y d }~qbd }~ww qbq|› d
}| j|ddD ]z\}}}|D ];}	| j ||	¡}
|| j |
¡ }||d kräz|  |
¡ W q© t	yã } zt
d|
› d|› ƒ W Y d }~q©d }~ww q©|D ]4}	| j ||	¡}|  |¡sz|  |¡ W qç t	y } zt
d	|› d|› ƒ W Y d }~qçd }~ww qçq¢d S )Nr   z/media/temp_files/developerr8   F)Útopdownr¬   zError deleting z: zError deleting folder z/media/temp_files/development)ÚosrÕ   r   ÚBASE_DIRÚwalkÚpathr¢  ÚgetmtimeÚremoveÚOSErrorrˆ   ÚlistdirÚrmdir)ra  rÕ   Úbase_dirÚfolder_pathÚage_thresholdrŠ   ÚrootÚdirsÚfilesr¼   Ú	file_pathÚfile_ager›   Údir_pathrM   rM   rP   Údelete_temp_filesí  sl   
 €ÿý
 €ÿýý
	 €ÿý €ÿýýõrs  c                  C   s   d} d}d}d}d}t jjdd ¡ }t jjdd ¡ }tj ¡  ¡  	 | d
8 } | dkr.d S zLt |› d| › ¡}|j	dkrLtjj
| |||j| |d n|d
8 }| dkrVW q#t |› d| › ¡}|j	dkrutjj
| d |||j| |d n|d
8 }W n   Y q$)NéA   é@   é   rd   ro   r<   r=   rl   Tr7   r   z/news/?page=r@   )rC   rF   rE   rD   rg   rf   )r+   rr   rs   rt   r   rv   rw   rp   rq   ru   ry   rT   )rŒ   Útotal_1Útotal_2Úurl_1Úurl_2Úcity_1Úcity_2rE  rM   rM   rP   Úget_news_data_1nh   s4   

 €îr}  c                  C   s6  t jjddd d… } | D ]Š}d}|j}|j}t|jdƒ}|jddd}|D ]o}z4|jdddj	}t
|d	d
}	|jdddj	}
|jdddd }|jdddj	}|jdddj	}|› |› }tj|dd}|jdkr^t|j	dƒ}|jddd}|r^|jdddd }d }|r³|› |› }tj|dd}|	› d| d¡d › }|jdkr³tt |j¡|d}|jddd}|jdd	dD ]}| ¡  qÁd }z	tjj|d}W n    tjj||	||
||||d  d!d"„ |jD ƒ¡ d#d$¡d%	}Y d}| d¡D ]L}|d }|rGtj|dd}|	› d| d¡d › }|jdkrGtt |j¡|d}z
tjj||d&}W n   tjj|||d'}d	}Y |j› |d< qû|r^d  d(d"„ |jD ƒ¡|_d|_| ¡  W n( ty‡ } zd	}t d)t!j! "¡  #d*¡› d+|j$› d,|ƒ W Y d }~nd }~ww |s—d	|_%t& &¡ |_'| ¡  q'qd S )-NFr­   r7   rH   r=  znews-block--wrapper)Úclass_rÝ   TrÀ   r¨  r¶   z
news-blockr·   ÚannouncerÒ   znews-block--rubricr±   r:   r@   rV  r2  zbasepage--cover-fullwidthr3  rÊ   r¸   r¹   r»   znews--main-content)r·   )rŽ   rY   c                 S   r"  rM   rM   r#  rM   rM   rP   r&  t  r'  z"parse_onh_news.<locals>.<listcomp>r(  r)  )	rÝ   r>   rF   r¨  rŽ   Úsummaryr‚   rÞ   r¯  )ÚnewsrŽ   )r  rŽ   rÞ   c                 S   s   g | ]}|d kr|› nd‘qS )r+  rY   rM   r#  rM   rM   rP   r&  †  s    ri   rj   z : in get news detail [ú
] error : )(r   rr   rs   rE   rF   r   rD   rÖ   r|   rT   r   rp   rq   ru   rØ   r   rÙ   rÚ   rÛ   ÚfindAllÚextractr   ry   r¢  r£  r~   r   rÞ   r¯  Úis_rephraser¤   r‡   rˆ   r‰   rŠ   r‹   rC   r®   rÕ   rç   )Úonh_news_dataÚonh_newsr§   rE   rF   r’   Ú	news_listr  rÝ   r>   r¨  rŽ   r€  r‚   Úc_urlrE  rÛ   rÞ   rì   Ú	image_urlrí   r@  rT   r¶   Ú
onh_n_newsÚ	add_imager2  Ú
text_imagerî   Úonh_n_news_imager›   rM   rM   rP   Úparse_onh_newsA  sž   



÷€€2€þ
€¹úr  zñparaphrase the following text, which news text on a real estate consulting website, into {language} language. Use HTML tags and headings h3-h6 if applicable. Maintain the tone of a professional real estate agent.
    Original Text: `{text}`
rË   zgpt-4o-minic              
   C   s¬   z*t td}|j| |d}|jjj|dddœd|dœgdd}d	|jd
 jjd dœW S  t	yU } zt
dtj ¡  d¡› d|› d|ƒ dd |› dœW  Y d }~S d }~ww )N)Úapi_key)rT   rÑ   ÚsystemzIYou are an experienced real estate consultant with deep market knowledge.)ÚrolerÛ   Úusergffffffæ?)ÚmodelÚmessagesÚtemperatureFr   )r§   rÛ   Ú	error_txtri   rj   z : in news text paraphrase [r‚  T)r3   r4   ÚformatÚchatÚcompletionsry   ÚchoicesÚmessagerÛ   r‡   rˆ   r‰   rŠ   r‹   )rT   rÝ   rÑ   Úpromptr”  Úclientr™  r›   rM   rM   rP   Úparaphrase_text—  s$   ÿÿü$€þrŸ  c                  C   sx   t jjddd ¡ } | j}| j}|r:t||ƒ}|d s1|d  dd¡ dd¡| _d| _|  	¡  d S d| _
|  	¡  d S d S )	NF)Úis_paraphraseÚparaphrase_errorr§   rÛ   z```htmlrY   z```T)r   rr   rs   rt   r¯  rÝ   rŸ  r~   r   r¤   r¡  )r  rT   rÝ   r˜   rM   rM   rP   Úonh_news_paraphrase¨  s   
ør¢  )`rz   rÙ   rƒ   r   rÕ   r‰   Úbs4r   Údjango.confr   Údata_scraper.modelsr   r   r   r   r   r	   r
   Údjango.core.files.imagesr   rp   Údjango.utils.textr   Údeveloper.modelsr   r   Údjango.core.files.baser   Údevelopment.modelsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   Úgeo_location.modelsr+   r,   r-   r.   Úmain.modelsr/   Údjango.contrib.gis.geosr0   Údjango.db.modelsr1   Úurllib.parser2   Úopenair3   ÚenvironÚEnvÚenvÚread_envÚstrr4   r5   r6   r   r«   rô   rö   rú   r  r(  rA  rO  rT  r_  rs  r}  r  ÚPARAPHRASE_PROMPTrŸ  r¢  rM   rM   rM   rP   Ú<module>   sb    $t



^ !    qRO;Q3!R