o
    gjI                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlZd dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZmZmZmZmZ d dlmZmZmZ d dlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d d	l&m'Z'm(Z(m)Z) d d
l*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 e2 Z3ej24  e35dZ6e35dZ7e35dZ8dZ9e9dfddZ:dZ;dde;fddZ<d&ddZ=dd Z>d d! Z?d"d# Z@d$d% ZAdS )'    N)OpenAI)BeautifulSoup)	ImageFile)ContentFile)OldDataOneNewHomesDataOneNewHomesDeveloperDataOneNewHomesDevelopmentsDataOneNewsHomesDeveloper)ContactPerson	DeveloperDeveloperTranslate)
AttachmentBuildingTypeBuildingTypeTranslateCurrencyDevelopmentDevelopmentTranslateFeatureFeatureTranslateFlat
KeyFeaturePaymentOptionPaymentOptionTranslate)CityCountryLocation)Language)log_func)Point)slugify
openai_keyopenai_pr_idopenai_org_ida  Paraphrase the following real estate description to make it more appealing and engaging for potential buyers, while maintaining all factual information. The description should be concise and clear:
            Original Description: "{text}"
            Paraphrased Description:gpt-4o-minic              
   C   s|   z%t td}|j| d}|jjj|dddd|dgdd}|jd	 jjW S  t	y= } zt
d
| W Y d }~d S d }~ww )Napi_key)textsystemIYou are an experienced real estate consultant with deep market knowledge.rolecontentuserffffff?modelmessagestemperaturer   paraphrase_text)r   r!   formatchatcompletionscreatechoicesmessager,   	Exceptionr   )r'   promptr0   clientr5   e r>   3/var/www/Entralon-Web-Backend/data_scraper/utils.pyr3   !   s$   
r3   zTranslate the following real estate description from {from_language} to {to_language}, while maintaining all factual information. The description should be concise and clear:
            Original Description: "{text}"
            Translated Description:englishpersianc              
   C   s   z0t td}|j||| d}|jjjddddd|dgdd	}t|jd
 jj	 |jd
 jj	W S  t
yL } zt| td| W Y d }~d S d }~ww )Nr%   )from_languageto_languager'   r$   r(   r)   r*   r-   r.   r/   r   translate_text)r   r!   r4   r5   r6   r7   printr8   r9   r,   r:   r   )r'   rB   rC   r;   r<   r5   r=   r>   r>   r?   rD   :   s(   
rD   enfac                 C   s.   ddl m}m}m} |||d}|| }|S )a  Translates the given text from the specified source language to the target language using the Google Translator.
    
    Args:
        text (str): The text to be translated.
        from_language (str, optional): The source language code. Defaults to "en".
        to_language (str, optional): The target language code. Defaults to "fa".
    
    Returns:
        str: The translated text.
    r   )GoogleTranslatorChatGptTranslatorDeeplTranslator)sourcetarget)deep_translatorrH   rI   rJ   	translate)r'   rB   rC   rH   rI   rJ   
translatortranslated_textr>   r>   r?   translate_text_by_googleP   s   
rQ   c                  C   s  d} |  d}d}i }t jd|||dd}|jdkrtjjdd	d
\}}| }|d D ]}z|d}	td|	  |d}
|d}t	|	dd}|d}d }zt j|  | dd}|jdkrpt
t|j| dd}W n ty } ztd| W Y d }~nd }~ww |d}tjj|d\}}|	|_||_||_||_|  tjj||d\}}|	f|_|f|_|
|_|  W q, ty } ztd| W Y d }~q,d }~ww d S d S )Nhttps://api.entralon.com/zapi/v1/developer?limit=1000 GET   headersdatatimeout   r@   rF   namecoderX   titlezstart developer:detailedDescriptiontargetLocationTallow_unicodeimagerY   z.pngr\   zimage save errorwebsiteslug)	developerlanguagezerror for loop: )requestsrequeststatus_coder   objectsget_or_createjsongetrE   r    r   ioBytesIOr,   r:   r   r\   rc   target_locationweb_addresssaver   rh   description)base_urlurlpayloadrW   responserj   _rX   itemr\   rw   rt   rh   rc   fimage_file_requestr=   rf   ri   dtr>   r>   r?   get_old_developerg   sX   







r   c                  C   s|   d} |  d}t ddD ]/}|  d| d}d}i }tjd|||d	d
}|jdkr;| }|d D ]	}tjj|d q1qd S )NrR   zapi/v1/developments?limit=1000      zapi/v1/developments?page=z	&limit=50rS   rT   rU   rV   rZ   rX   )rX   )rangerk   rl   rm   rp   r   rn   r7   )rx   ry   irz   rW   r{   rX   r}   r>   r>   r?   download_old_development   s   

r   c            R      C   s  d} t jjddd\}}t jjddd\}}t jjddd\}}t jjd	d
d\}}tj D ]I}|j}z,|d}t|dd}	tjj|	d\}
}||
_	t
d|  |d}|d}z|d}W n   d}Y |d}z|d}W n   d}Y |d}z|d}W n   d}Y |d
}z|d}W n   d}Y ztjj||
||	|d W n   Y ztjj||
||	|d W n   Y ztjj||
||	|d W n   Y ztjj||
||	|d W n   Y |d}ztjj|dd}W n   d }Y ||
_|d}z|dd}t|dd}tjj|d\}}W n   d }Y ||
_|d}||
_|dd}| dkrMd }d }d}n|d d }z	tj|d}W n   d }Y d}||
_||
_||
_|d }|D ]} | d!}!t|!dd}"| d}#z|#dd!}$t|$dd}%W n   d}$d}%Y z|#dd!}&t|&dd}'W n   d}&d}'Y z|#d
d!}(t|(dd})W n   d}(d})Y ztjj|!|"d"\}*}W n   d }*Y |*d ur:ztjj|*|!|"||!d# W n   Y ztjj|*|$|%||$d# W n   Y ztjj|*|(|)||(d# W n   Y ztjj|*|&|'||&d# W n   Y tjj|
|*d$ qu|d%}+|+D ]G},|, }-t|-dd}.ztjj|-|.d"\}/}W n   d }/Y |/d urztjj|/|-|.||-d& W n   Y z	|
j|/ W qC   Y qCqC|d'}0|0D ]E}1|1}2t|2dd}3ztjj|2|3d"\}4}W n   d }4Y |4d urzt jj|4|2|3||2d( W n   Y z	|
j!|4 W q   Y qq|d)}5|5D ]?}6|6d*d}7|6d+d}8t"|6d,ddk}9|6d-d}:|6d.d};t#jj|
|7d/\}6}|:|6_$|;|6_%|8|6_&|9|6_'|6(  q|d0}<|<D ]z}=|=d1d}>|=d2}?d }@z%t)j|  |? d3d4}A|?*d5d6 }B|Aj+d7krYt,t-.|Aj/|B d8}@W n   Y d}C|> d9v rkd}Cn'|> d:v rud;}Cn|> d<v rd=}Cn|> d>v rd}Cn	|> d?v rd@}Ct0jj|@|@||	|C|
ddA q&|d*}7|d}||
_&|dB|
_1|dC|
_2z|dD}D|Dd urC|DdE}E|DdF}Fz|DdG}Gt3jj|Gd8\}H}W n   d }HY z|DdH}I|DdI}Jt4jj|I|Jd\}K}W n   d }KY |DdJ}L|DdK}M|DdL}N|DdM}Ot5jjt6|Ed |Ed dN\}P}|F|P_|L|P_7|K|P_8|H|P_9|M|P_:|O|P_;|N|P_<|P(  |P|
_=W n t>y\ }Q zt
dO|Q W Y d }Q~Qnd }Q~Qww |
(  W q/ t>yy }Q z
t
|Q W Y d }Q~Qq/d }Q~Qww d S )PNrR   r@   rF   r[   rA   rG   russianruarabicarr^   Tra   rg   zstart development:translationsrw   rS   )rj   developmentr^   rh   rw   currencyacronymr]   	basePriceri   addresscompletionDatezready to mover      z%Yr   keyFeaturesfeature)r\   rh   )r   r\   rh   rj   rw   )r   r   buildingType)building_typer\   rh   rj   rw   paymentOptions)payment_optionr\   rh   rj   rw   flatsbedRoomcurrentPriceisAvailablesquarepricePerMeter)r   bedrooms_num
attachmentcategoryfile
   rd   /rZ   re   )exterior)interior   )	amenities   )zfloor plansz
floor plan	floorplan
floorplans)z	site planz
site planssiteplan	siteplans   )r   rc   r^   altr   r   typeisPublishedreferenceIdlocationcoordinatesformattedAddresscitycountrycountryCodestreetstreetNumberplace_idzipCode)r   zlocation error)?r   rn   ro   r   allrX   rq   r    r   r^   rE   r   r7   r   default_currencyr   ri   r   lowerdatetimestrptimecompleted_datecompleted_atcompleted_statusr   r   r   r   r   r   addr   r   r   intr   areaprice_per_meter
base_price	is_activerv   rk   splitrm   r   rr   rs   r,   r   is_publishedreference_idr   r   r   r   r   r   r   street_numberzip_coder   r   r:   )Rrx   language_enr|   language_falanguage_rulanguage_arrX   r}   r\   rh   r   r   rF   description_enrG   description_far   description_rur   description_ar	currency_r   r   developer_namedeveloper_slugri   r   r   r   r   r   r   r   feature_namefeature_slugfeature_translationsfeature_name_fafeature_slug_fafeature_name_rufeature_slug_rufeature_name_arfeature_slug_arfeature_buildingTypesr   building_type_namebuilding_type_slugbuilding_type_r   r   payment_option_namepayment_option_slugpayment_option_r   flatr   r   r   r   r   attachmentsr   r   r   r~   r   	file_namefile_categoryr   r   address_r   city_r   r   country_street_r   r   r   	location_r=   r>   r>   r?   get_old_developments   s  





















	











 & $  r   c                  C   s
  t dddu} t| }|D ]d}t| || }t| tjj|d\}}|d|_|d|_	|d|_
|d|_|d	d
|_|d|_|d|_|d|_|d|_|d|_|d|_|d|_|  qW d    d S 1 s~w   Y  d S )Nzcurrencies.jsonzutf-8)encodingr   r\   demonymmajorSinglemajorPluralISOnumr   symbolsymbolNativeminorSingleminorPlural	ISOdigitsdecimals
numToBasic)openrp   loadrE   r   rn   ro   rq   r\   r   major_singlemajor_pluraliso_numr   symbol_nativeminor_singleminor_plural
iso_digitsdecimal_digitsnum_to_basicrv   )r~   dr}   rX   cr|   r>   r>   r?   add_currencyr  s,   

"r  )rF   rG   )Br   rp   timerr   osopenair   environrk   bs4r   django.core.files.imagesr   django.core.files.baser   data_scraper.modelsr   r   r   r	   r
   developer.modelsr   r   r   development.modelsr   r   r   r   r   r   r   r   r   r   r   r   geo_location.modelsr   r   r   main.modelsr   
main.utilsr   django.contrib.gis.geosr   django.utils.textr    Envenvread_envstrr!   r"   r#   PARAPHRASE_PROMPTr3   TRANSLATE_PROMPTrD   rQ   r   r   r   r  r>   r>   r>   r?   <module>   sD    8




- R