import datetime import html import io import json import os import random import re import time from django.conf import settings import requests from data_scraper.utils import translate_text_by_google from development.models import Attachment, BuildingType, BuildingTypeTranslate, ChangeRate, Currency, Development, DevelopmentDetail, DevelopmentOffer, DevelopmentOfferTranslate, DevelopmentTranslate, Faq, FaqCategory, FaqCategoryTranslate, FaqTranslate, Feature, FeatureTranslate, FilterSeoText, FilterSeoTextTranslate, Flat, FlatTranslate, KeyFeature, ONHArea, ONHDistrict, ONHPostcode, ONHStation, ONHZone, PaymentOption, PaymentOptionTranslate, PaymentPlan, PaymentPlanTranslate, RankElement, RankElementTranslate from development.utils import flat_bedroom_type from geo_location.models import City, Country, PublicFacility, StreetView from main.models import Language from django.db.models import Q, Count, Max, Min import textwrap from django.core.files.images import ImageFile from PIL import Image import urllib.request from bs4 import BeautifulSoup from django.utils.text import slugify from openai import OpenAI import environ from main.utils import log_func env = environ.Env() environ.Env.read_env() from geopy.distance import geodesic openai_key = env.str('openai_key') openai_pr_id = env.str('openai_pr_id') openai_org_id = env.str('openai_org_id') PARAPHRASE_PROMPT = """paraphrase the following text, which describes an apartment or building project on a real estate consulting website, into {language} language. Use HTML tags and headings h3-h6 if applicable. Maintain the tone of a professional real estate agent, keeping in mind that the property is located in {address} and has access to {stations} stations. Original Description: `{text}` """ def paraphrase_description(text, name, address, stations, language="english", prompt=PARAPHRASE_PROMPT, model="gpt-4o-mini"): try: client = OpenAI( api_key=openai_key, ) prompt = prompt.format(text=text, name=name, address=address, stations=stations, language=language) chat = client.chat.completions.create( model = model, messages = [{"role": "system", "content": "You are an experienced real estate consultant with deep market knowledge."}, {"role": "user", "content": prompt}], temperature = 0.7 ) return {"error": False, "content":chat.choices[0].message.content, "error_txt":None} except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in developer description paraphrase [{name}] error : ', e) return {"error": True, "content":None, "error_txt":f"{e}"} def development_description(): developments = DevelopmentTranslate.objects.filter(is_paraphrase=False, paraphrase_error=False)[:10] for development_t in developments: try: development = development_t.development stations = ", ".join(development.onh_station.all().values_list("name", flat=True)) try: des = development.translates.filter(language__code="en").first().description except: des = None if des: description = paraphrase_description(des, development.title, development.address, stations, development_t.language.name) if description['error']: development_t.paraphrase_error = True development_t.error = description["error_txt"] development_t.save() print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in development description paraphrase [{development_t.title}:{development_t.id}] error : {description["error_txt"]}') elif description['content']: development_t.description = description['content'].replace("html", "").replace("```", "") development_t.is_paraphrase = True development_t.last_paraphrase = time.time() development_t.save() else: development_t.paraphrase_error = True development_t.save() print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in development description paraphrase [{development_t.title}:{development_t.id}] error : development description is None') except Exception as e: development_t.paraphrase_error = True development_t.error = f"{e}" development_t.save() print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in development description paraphrase [{development_t.title}:{development_t.id}] error : ', e) SUMMARY_PROMPT = """Summarize the following real estate description in a concise and engaging manner in {language} language, only 500 character, while maintaining all factual information, without HTML tags, ensuring it captivates the user and encourages further reading. The summary should be concise and clear. Original Description : {text}""" def summarize_description(text, id, language="english", prompt=SUMMARY_PROMPT, model="gpt-4o-mini"): try: client = OpenAI( api_key=openai_key, ) prompt = prompt.format(text=text, language=language) chat = client.chat.completions.create( model = model, messages = [{"role": "system", "content": "You are an experienced real estate consultant with deep market knowledge."}, {"role": "user", "content": prompt}], temperature = 0.7 ) print(chat.choices) return {"error": False, "content":chat.choices[0].message.content, "error_txt":None} except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in development description summarize [{id}] error : ', e) return {"error": True, "content":None, "error_txt":f"{e}"} def development_summarize(): developments = DevelopmentTranslate.objects.filter(is_paraphrase=True, is_summarize=False, summarize_has_error=False)[:10] for development_t in developments: try: des = development_t.description if des: description = summarize_description(des, development_t.id, development_t.language.name) if description['error']: development_t.summarize_has_error = True development_t.summarize_error = description["error_txt"] development_t.save() print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in development description paraphrase [{development_t.title}:{development_t.id}] error : {description["error_txt"]}') elif description['content']: development_t.summary = description['content'] development_t.is_summarize = True development_t.last_paraphrase = time.time() development_t.save() else: development_t.summarize_has_error = True development_t.save() print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in development description paraphrase [{development_t.title}:{development_t.id}] error : development description is None') except Exception as e: development_t.summarize_has_error = True development_t.error = f"{e}" development_t.save() print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in development description paraphrase [{development_t.title}:{development_t.id}] error : ', e) def add_currencies_rate(): base_url = "https://v6.exchangerate-api.com/v6/3194c7590734b504e82d1dfa/latest/" base_currencies = Currency.objects.filter(developments__in=Development.objects.filter(is_active=True, is_deleted=False, country_id__in=[2,3])).distinct() for base_currency in base_currencies: response = requests.get(base_url + base_currency.code, timeout=10) data = response.json() conversion_rates = data["conversion_rates"] for currency, rate in conversion_rates.items(): try: c_t = Currency.objects.get(code=currency) ChangeRate.objects.create(from_currency=base_currency, to_currency=c_t, rate=rate) if base_currency.code == "GBP": c_t.gbp_rate = rate elif base_currency.code == "AED": c_t.aed_rate = rate c_t.save() except Exception as e: print(f"Error: {currency} not found in the database.") def translate_payment_option(): languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") for po in PaymentOption.objects.filter(translated=False)[:10]: try: l = PaymentOptionTranslate.objects.get(language=base_language, payment_option=po) except: l = PaymentOptionTranslate.objects.create(language=base_language, payment_option=po, name=po.name, slug=po.slug) error = False for language in languages: try: name = translate_text_by_google(po.name, to_language=language.code) slug = po.slug try: pot = PaymentOptionTranslate.objects.get(language=language, payment_option=po) pot.name = name pot.slug = slug pot.save() except: PaymentOptionTranslate.objects.create(language=language, payment_option=po, name=name, slug=slug) except Exception as e: error = True print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {e}') if not error: po.translated = True po.save() def translate_features(): languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") for kf in Feature.objects.filter(translated=False)[:10]: try: l = FeatureTranslate.objects.get(language=base_language, feature=kf) except: l = FeatureTranslate.objects.create(language=base_language, feature=kf, name=kf.name, slug=kf.slug) error = False for language in languages: try: name = translate_text_by_google(kf.name, to_language=language.code) slug = kf.slug try: kft = FeatureTranslate.objects.get(language=language, feature=kf) kft.name = name kft.slug = slug kft.save() except: FeatureTranslate.objects.create(language=language, feature=kf, name=name, slug=slug) except Exception as e: error = True print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {e}') if not error: kf.translated = True kf.save() def translate_building_type(): languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") for kf in BuildingType.objects.filter(translated=False)[:10]: try: l = BuildingTypeTranslate.objects.get(language=base_language, building_type=kf) except: l = BuildingTypeTranslate.objects.create(language=base_language, building_type=kf, name=kf.name, slug=kf.slug) error = False for language in languages: try: name = translate_text_by_google(kf.name, to_language=language.code) slug = kf.slug try: kft = BuildingTypeTranslate.objects.get(language=language, building_type=kf) kft.name = name kft.slug = slug kft.save() except: BuildingTypeTranslate.objects.create(language=language, building_type=kf, name=name, slug=slug) except Exception as e: error = True print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {e}') if not error: kf.translated = True kf.save() def translate_rank_element(): languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") for kf in RankElement.objects.filter(translated=False)[:10]: try: l = RankElementTranslate.objects.get(language=base_language, rank_element=kf) except: l = RankElementTranslate.objects.create(language=base_language, rank_element=kf, name=kf.name, slug=kf.slug) error = False for language in languages: try: name = translate_text_by_google(kf.name, to_language=language.code) slug = kf.slug try: kft = RankElementTranslate.objects.get(language=language, rank_element=kf) kft.name = name kft.slug = slug kft.save() except: RankElementTranslate.objects.create(language=language, rank_element=kf, name=name, slug=slug) except Exception as e: error = True print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {e}') if not error: kf.translated = True kf.save() def translate_development_offer(): languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") for kf in DevelopmentOffer.objects.filter(translated=False)[:10]: try: l = DevelopmentOfferTranslate.objects.get(language=base_language, development_offer=kf) except: l = DevelopmentOfferTranslate.objects.create(language=base_language, development_offer=kf, text=kf.text) error = False for language in languages: try: text = translate_text_by_google(kf.text, to_language=language.code) try: kft = DevelopmentOfferTranslate.objects.get(language=language, development_offer=kf) kft.text = text kft.save() except: DevelopmentOfferTranslate.objects.create(language=language, development_offer=kf, text=text) except Exception as e: error = True print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {e}') if not error: kf.translated = True kf.save() def translate_payment_plan(): languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") for kf in PaymentPlan.objects.filter(translated=False)[:10]: try: l = PaymentPlanTranslate.objects.get(language=base_language, payment_plan=kf) except: l = PaymentPlanTranslate.objects.create(language=base_language, payment_plan=kf, title=kf.title, slug=kf.slug) error = False for language in languages: try: title = translate_text_by_google(kf.title, to_language=language.code) slug = kf.slug try: kft = PaymentPlanTranslate.objects.get(language=language, payment_plan=kf) kft.title = title kft.slug = slug kft.save() except: PaymentPlanTranslate.objects.create(language=language, payment_plan=kf, title=title, slug=slug) except Exception as e: error = True print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {e}') if not error: kf.translated = True kf.save() def translate_development(): languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") for development in Development.objects.annotate(tr_num=Count('translates')).filter(Q(translated=False) | Q(tr_num__lt=languages.count()+1))[:10]: try: l = development.translates.get(language=base_language) except: l = DevelopmentTranslate.objects.create(development=development, address=development.address, language=base_language, title=development.title, slug=development.slug) if l is not None: try: address = l.address if l.address is not None else development.address description = l.description neighborhood = l.neighborhood features_description = l.features_description for language in languages: t_title = None t_slug = None t_description = None t_features_description = None t_neighborhood = None t_address = None t_title = development.title t_slug = development.slug if address is not None: t_address = translate_text_by_google(address, to_language=language.code) if description is not None and description != "": lines = [] try: lines = textwrap.wrap(description, 300, break_long_words=False) except Exception as e: print("description", e) des = [] for line in lines: if line is not None: des.append(translate_text_by_google(line, to_language=language.code)) t_description = " ".join(des) if features_description is not None and features_description != "": lines = [] try: lines = textwrap.wrap(features_description, 300, break_long_words=False) except Exception as e: print("features_description", e) fdes = [] for line in lines: if line is not None: fdes.append(translate_text_by_google(line, to_language=language.code)) t_features_description = " ".join(fdes) if neighborhood: try: t_neighborhood = translate_text_by_google(neighborhood, to_language=language.code) except Exception as e: print("neighborhood", e) try: try: development_t = DevelopmentTranslate.objects.get(language=language, development=development) except: development_t = DevelopmentTranslate(language=language, development=development) development_t.title=t_title development_t.slug=t_slug development_t.address=t_address development_t.description=t_description development_t.neighborhood=t_neighborhood development_t.features_description=t_features_description development_t.save() except Exception as e: print(e) development.translated = True development.save() except Exception as e: print(e) def get_panorama_detail(): import streetview for development in Development.objects.filter(panorama_check=False, location__isnull=False)[:1]: try: location = development.location ps = streetview.search_panoramas(lat=location.coordinates.y, lon=location.coordinates.x) for p in ps: if p.pano_id is not None: sv, _ = StreetView.objects.get_or_create(location=location, development = development, panorama_id=p.pano_id) sv.lat = p.lat sv.lon = p.lon sv.heading = p.heading sv.pitch = p.pitch sv.roll = p.roll sv.date = p.date sv.elevation = p.elevation sv.save() development.panorama_check = True development.save() except Exception as e: print(e) def get_street_view_images(): sv = StreetView.objects.filter(image_downloaded=False).first() num = random.randint(1, 3) dev = sv.development bd = settings.BASE_DIR zoom = 2 try: new_im = Image.new('RGB', (2048, 1024)) x_offset = 0 for x in range(0, 2**zoom): y_offset = 0 for y in range(0, 2**(zoom-1)): url = f"https://cbk{num}.google.com/cbk?output=tile&panoid={sv.panorama_id}&zoom={zoom}&x={x}&y={y}" img_name = f"{bd}/downloads/{x}{y}.jpg" urllib.request.urlretrieve(url, img_name) img = Image.open(img_name) new_im.paste(img, (x_offset,y_offset)) y_offset += 512 x_offset += 512 title = f"panorama image from {dev.title}" file_name = f"panorama-zoom{zoom}-{sv.panorama_id}" image_bytes = io.BytesIO() new_im.save(image_bytes, format="JPEG") image_bytes.seek(0) image = ImageFile(image_bytes, name=f"{file_name}.jpg") try: Attachment.objects.get(file_name=file_name, development=dev, category=7) except: try: Attachment.objects.create(file_name=file_name, development=dev, title=title, alt=title, file=image, image=image, category=7) except Exception as e: print(e) sv.image_downloaded=True sv.save() except Exception as e: print(e) ## # Here are the types of apartment units translated into Persian, Arabic, ru, and en: # #| **Type** | **Persian (فارسی)** | **Arabic (العربية)** | **ru (Русский)** | **en** | #|------------------------------|---------------------------------|-------------------------------|-------------------------------|------------------------------| #| **Studio Apartment** | آپارتمان استودیو | شقة استوديو | Квартира-студия | Studio Apartment | #| **One-bedroom Apartment** | آپارتمان یک خوابه | شقة بغرفة نوم واحدة | Однокомнатная квартира | One-bedroom Apartment | #| **Two-bedroom Apartment** | آپارتمان دو خوابه | شقة بغرفتين نوم | Двухкомнатная квартира | Two-bedroom Apartment | #| **Penthouse** | پنت‌هاوس | بنتهاوس | Пентхаус | Penthouse | #| **Loft** | آپارتمان لافت | شقة لوفت | Лофт | Loft | #| **Duplex** | آپارتمان دوبلکس | شقة دوبلكس | Дуплекс | Duplex | #| **Triplex** | آپارتمان تریپلکس | شقة تريبلكس | Триплекс | Triplex | #| **Garden Apartment** | آپارتمان باغ | شقة مع حديقة | Квартира с садом | Garden Apartment | #| **Basement Apartment** | آپارتمان زیرزمین | شقة تحت الأرض | Квартира в подвале | Basement Apartment | #| **High-rise Apartment** | آپارتمان برج بلند | شقة في ناطحة سحاب | Высотная квартира | High-rise Apartment | #| **Low-rise Apartment** | آپارتمان برج کوتاه | شقة في مبنى منخفض | Низкоэтажная квартира | Low-rise Apartment | #| **Condominium (Condo)** | آپارتمان کاندو | شقة تملكية | Кондоминиум | Condominium (Condo) | #| **Serviced Apartment** | آپارتمان خدماتی | شقة مفروشة بخدمات | Обслуживаемая квартира | Serviced Apartment | #| **Co-op Apartment** | آپارتمان مشارکتی | شقة تعاونية | Кооперативная квартира | Co-op Apartment | # # Let me know if you need further clarification! # # # # apartments = { "Studio Apartment": { "fa": "آپارتمان استودیو", "ar": "شقة استوديو", "ru": "Квартира-студия", "en": "Studio Apartment" }, "One-bedroom Apartment": { "fa": "1 خوابه", "ar": "بغرفة نوم 1", "ru": "1 комнатная", "en": "1 bedroom" }, "Two-bedroom Apartment": { "fa": "آپارتمان دو خوابه", "ar": "شقة بغرفتين نوم", "ru": "Двухкомнатная квартира", "en": "Two-bedroom Apartment" }, "Penthouse": { "fa": "پنت‌هاوس", "ar": "بنتهاوس", "ru": "Пентхаус", "en": "Penthouse" }, "Loft": { "fa": "آپارتمان لافت", "ar": "شقة لوفت", "ru": "Лофт", "en": "Loft" }, "Duplex": { "fa": "دوبلکس", "ar": "دوبلكس", "ru": "Дуплекс", "en": "Duplex" }, "Triplex": { "fa": "آپارتمان تریپلکس", "ar": "شقة تريبلكس", "ru": "Триплекс", "en": "Triplex" }, "Garden Apartment": { "fa": "آپارتمان باغ", "ar": "شقة مع حديقة", "ru": "Квартира с садом", "en": "Garden Apartment" }, "Basement Apartment": { "fa": "آپارتمان زیرزمین", "ar": "شقة تحت الأرض", "ru": "Квартира в подвале", "en": "Basement Apartment" }, "High-rise Apartment": { "fa": "آپارتمان برج بلند", "ar": "شقة في ناطحة سحاب", "ru": "Высотная квартира", "en": "High-rise Apartment" }, "Low-rise Apartment": { "fa": "آپارتمان برج کوتاه", "ar": "شقة في مبنى منخفض", "ru": "Низкоэтажная квартира", "en": "Low-rise Apartment" }, "Condominium (Condo)": { "fa": "آپارتمان کاندو", "ar": "شقة تملكية", "ru": "Кондоминиум", "en": "Condominium (Condo)" }, "Serviced Apartment": { "fa": "آپارتمان خدماتی", "ar": "شقة مفروشة بخدمات", "ru": "Обслуживаемая квартира", "en": "Serviced Apartment" }, "Co-op Apartment": { "fa": "آپارتمان مشارکتی", "ar": "شقة تعاونية", "ru": "Кооперативная квартира", "en": "Co-op Apartment" } } def translate_flats(): languages = Language.objects.all() lc = languages.count() for flat in Flat.objects.annotate(tr_num=Count('translates')).filter(tr_num__lt=lc)[:60]: for language in languages: try: try: l = flat.translates.get(language=language) except: l = FlatTranslate.objects.create(flat=flat, language=language) l.name = flat_bedroom_type(flat.bedrooms_num, language.code) l.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} flat {flat.id} translate to {language.name} create:', e) def set_development_geo_json(): lc = time.time() - 3600*12 currency_1 = Currency.objects.get(code="GBP") currency_2 = Currency.objects.get(code="AED") for d in Development.objects.filter(Q(geo_json__isnull=True) | Q(last_create__lte=lc)): try: flats = list(d.flats.all().values("bathrooms_num", "bedrooms_num", "is_sold_out", "area", "price_per_meter", "base_price")) num = [] for flat in d.flats.all(): num.append(flat.bedrooms_num) num.sort() bedrooms_num = ",".join([f"{item if item > 0 and item < 10000 else flat_bedroom_type(item, lang='en')}" for item in num]) try: developer = {"name":d.developer.name, "slug": f"developer-{d.developer.slug}", "image": f"{d.developer.image}", "id":d.developer.id} except: developer = {} try: currency = {"name":d.default_currency.name, "code": d.default_currency.code, "symbol": d.default_currency.symbol} except: if d.country_id == 2: d.default_currency = currency_1 currency = {"name":currency_1.name, "code": currency_1.code, "symbol": currency_1.symbol} else: d.default_currency = currency_2 currency = {"name":currency_2.name, "code": currency_2.code, "symbol": currency_2.symbol} geo_json = { "type": "Feature", "id": d.id, "properties": { "id": d.id, "title": d.title, "slug": f"property-{d.slug}", "base_price": d.base_price, "address": d.address, "area": d.area, "min_area": d.min_area, "max_area": d.max_area, "floor": d.floor, "flat": d.flat, "is_featured": d.is_featured, "completed_date": d.completed_date, "is_ready_to_move": d.is_ready_to_move, "is_sold_out": d.is_sold_out, "image": f"{d.image}", "flats": flats, "bedrooms_num": bedrooms_num, "developer":developer, "currency": currency, }, "geometry": { "type": "Point", "coordinates": [ d.coordinates.x, d.coordinates.y ] } } d.geo_json = geo_json d.last_create = time.time() d.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {d.id} geo json:', e) def set_development_translate_geo_json(): lc = time.time() - 3600*12 currency_1 = Currency.objects.get(code="GBP") currency_2 = Currency.objects.get(code="AED") for dt in DevelopmentTranslate.objects.filter(Q(geo_json__isnull=True) | Q(last_create__lte=lc)): d = dt.development language = dt.language try: flats = list(d.flats.all().values("bathrooms_num", "bedrooms_num", "is_sold_out", "area", "price_per_meter", "base_price")) num = [] for flat in d.flats.all(): num.append(flat.bedrooms_num) num.sort() bedrooms_num = ",".join([f"{item if item > 0 and item < 10000 else flat_bedroom_type(item, lang=language.code)}" for item in num]) try: developer = {"name":d.developer.name, "slug": f"developer-{d.developer.slug}", "image": f"{d.developer.image}", "id":d.developer.id} except: developer = {} try: currency = {"name":d.default_currency.name, "code": d.default_currency.code, "symbol": d.default_currency.symbol} except: if d.country_id == 2: d.default_currency = currency_1 currency = {"name":currency_1.name, "code": currency_1.code, "symbol": currency_1.symbol} else: d.default_currency = currency_2 currency = {"name":currency_2.name, "code": currency_2.code, "symbol": currency_2.symbol} d.save() geo_json = { "type": "Feature", "id": d.id, "properties": { "id": d.id, "title": d.title, "slug": f"property-{d.slug}", "base_price": d.base_price, "address": dt.address, "area": d.area, "min_area": d.min_area, "max_area": d.max_area, "floor": d.floor, "flat": d.flat, "is_featured": d.is_featured, "completed_date": d.completed_date, "is_ready_to_move": d.is_ready_to_move, "is_sold_out": d.is_sold_out, "image": f"{d.image}", "flats": flats, "bedrooms_num": bedrooms_num, "developer":developer, "currency": currency, }, "geometry": { "type": "Point", "coordinates": [ d.coordinates.x, d.coordinates.y ] } } dt.geo_json = geo_json dt.last_create = time.time() dt.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {d.id} geo json:', e) def set_ready_to_move(): now = datetime.datetime.now() yesterday = now - datetime.timedelta(days=1) Development.objects.filter(completed_at__year__lt=now.year).update(is_ready_to_move=True, completed_status=1) Development.objects.filter(completed_date__icontains="Ready to move").update(is_ready_to_move=True, completed_status=1) complete_array =["finished", 'ready_to_mode', 'ready-to-mode', "ready to move", "Ready To Move", "ReadyToMove", "Ready to move"] Development.objects.filter(completed_date__in=complete_array).update(is_ready_to_move=False, completed_status=0) Flat.objects.filter(base_price=0, price_per_meter=0).update(is_sold_out=True) Development.objects.annotate(f_base_price=Max('flats__base_price')).filter(f_base_price=0).update(is_sold_out=True) Development.objects.annotate(f_area=Max('flats__area')).filter(f_area=0).update(is_sold_out=True) Development.objects.filter(flats__isnull=True, created_at__lt=yesterday).update(is_sold_out=True) Development.objects.filter(attachments__isnull=True).update(is_active=False, is_published=False) for country in Country.objects.all(): if country.coordinates is not None: Development.objects.filter(country=country).exclude(coordinates__intersects=country.coordinates).update(is_active=False, is_published=False) for a in Attachment.objects.filter(type=0): if not os.path.exists(a.image.path): a.delete() def get_onh_station_json(): station = ONHStation.objects.filter(json__isnull=True).first() if station: name = station.name city = station.city code = station.code if city.name == "Dubai": base_url = "https://1newhomes.ae/new-homes/" url = f"https://1newhomes.ae/assets/components/msearch2/action-search.php" else: base_url = "https://1newhomes.com/new-homes/" url = f"https://1newhomes.com/assets/components/msearch2/action-search.php" try: req_co = requests.get(base_url, timeout=10) if req_co.status_code == 200: co = req_co.cookies PHPSESSID = co['PHPSESSID'] html_content = req_co.text soup = BeautifulSoup(html_content, 'html.parser') script_tag = soup.find('script', text=lambda t: t and 'mse2Config' in t) script_content = script_tag.string.replace(",\n", ", ").replace(",\r", ", ").replace("\t","").replace(" "," ").replace(" "," ") mse2Config_match = re.search(r'mse2Config\s*=\s*({.*})', script_content, re.DOTALL) key = "30e35bbdbc0d5a572a05f1b709551ada3d2b5e7b" if mse2Config_match: mse2Config = mse2Config_match.group(1) mse2Config_json = json.loads(mse2Config) key = mse2Config_json.get('key') payload = f"station={code}&action=filter&pageId=2923&key={key}" if city.name == "Dubai": headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''linkchain=/ ### /news/a-15-quarter-climb-in-dubai-real-estate-prices ### /developers ### /emaar-properties-dubai/ ### / ### / ### / ### /new-homes/ ### /the-address-jumeirah-resort-and-spa-dubai ### /emaar-properties-dubai/ ### /the-address-jumeirah-resort-and-spa-dubai ### / ### /new-homes/ ### /prescott-real-estate-dubai/ ### /developers ### /new-homes/ ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /new-homes/page-2 ### /berton-dubai ### /berton-dubai ### /riviera-dubai ### /riviera-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai/floorplans ### / ### /the-mansions-dubai ### / ### /w1nner-tower-winner-tower-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/page-100 ### /samana-lake-views-dubai ### / ### /golf-grand1-dubai ### /new-homes/ ### /new-homes/page-100 ### /the-legends-villas-dubai ### / ### /claydon-house-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /golf-grand1-dubai ### /emaar-properties-dubai/ ### /new-homes/ ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-20 ### /new-homes/property-studio/page-30 ### /damac-properties-dubai/ ### /new-homes/property-studio/page-30 ### /merano-tower-dubai ### /sky-hills-residences-2-dubai ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-30 ### /golf-grand1-dubai ### /camelia-villas-dubai ### /new-homes/property-studio/page-30 ### /liv-waterside-dubai ### /new-homes/ ### /elvira-dhe-dubai ### /new-homes/ ### /the-vybe-dubai ### /the-vybe-dubai ### /sky-hills-residences-2-dubai ### /new-homes/ ### /j-one-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /j-one-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /imperial-avenue-dubai ### /new-homes/ ### /damac-hills-utopia-dubai ### /new-homes/apartments/property-studio/ ### /j-one-dubai ### /damac-hills-utopia-dubai ### /al-jawhara-tower-dubai ### /al-jawhara-tower-dubai/gallery ### /new-homes/apartments/property-studio/ ### /new-homes/apartments/property-studio/ ### /moonsa-residences-2-dubai ### /new-homes/apartments/property-studio/ ### /linden-residences-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /j-one-dubai/gallery ### /' ### / ### /new-homes/ ### /mariane-tower-dubai ### /developers ### /samana-developers-dubai/ ### / ### /imperial-avenue-dubai ### / ### /peninsula-dubai ### / ### /damac-lagoons-morocco-2-dubai ### / ### /damac-casa-dubai ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /parkway-dubai ### /new-homes/ ### /parkway-dubai ### /parkway-dubai/gallery ### /parkway-dubai ### /new-homes/apartments/ ### /new-homes/ ### /riviera-dubai ### / ### /parkway-dubai ### / ### / ### / ### /new-homes/ ### /new-homes/station-jumeirah-beach-residence/ ### /alta-real-estate-developments-dubai/ ### /mr-c-residences-jumeirah-dubai ### / ### / ### /city-walk-northline-dubai ### /parkside-hills-dubai ### /new-homes/ ### /peninsula-dubai ### /parkside-hills-dubai ### / ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /beverly-hills-drive-dubai ### /new-homes/ ### /ag-square-dubai ### /new-homes/ ### / ### /new-homes/ ### /mina-dubai ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### / ### /zada-tower-dubai ### /new-homes/ ### /new-homes/ ### /linden-residences-dubai ### /linden-residences-dubai ### /new-homes/ ### /beach-walk-3-dubai ### / ### /harbour-gate-dubai ### /new-homes/ ### /park-gate-residences-dubai ### /new-homes/ ### /merano-tower-dubai ### / ### /vanguard-by-franсk-muller-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-by-address-resorts-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /new-homes/ ### /davinci-tower-dubai ### /new-homes/ ### /mudon-al-ranim-1-dubai ### /new-homes/ ### /kaya-dubai ### /kaya-dubai ### /new-homes/; _ga_SVS4S7PFEJ=GS1.1.1730685094.108.1.1730685108.0.0.0; _ga=GA1.1.415606180.1724083972; _ym_uid=1719929312736223957; _ym_d=1724083975; cookie_agree=1; PHPSESSID={PHPSESSID}; utm=; _ym_isad=1; _ym_visorc=w''' } else: headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''_ga_JCEVRN83E0=GS1.1.1730680201.285.1.1730681307.44.0.0; _ga=GA1.1.65359444.1719929309; _ym_uid=1719929310452810339; _ym_d=1719929310; _fbp=fb.1.1722030370979.323930974364926; _ga_JCEVRN83E0=deleted; _gcl_au=1.1.962196760.1727727415; linkchain=/ ### /hendon-waterside-london ### / ### /eastman-village-london ### /new-homes/ ### /almax-group-london/ ### /new-homes/year-finished/ ### /1a-st-johns-wood-park-london ### /1a-st-johns-wood-park-london ### /new-homes/?property=1-bedroom&year=Finished ### / ### /new-homes/ ### / ### /new-homes/ ### / ### /developers ### / ### /new-homes/ ### / ### /new-homes/ ### /"https:////1newhomes.com//almax-group-london///" ### /new-homes/ ### / ### /invest ### /first-time-buyers ### /first-time-buyers ### /first-time-buyers ### /the-waterline-london ### /new-homes/options-concierge-service/ ### / ### / ### /luxury-properties ### / ### / ### /developers ### /new-homes/ ### / ### /new-homes/ ### /kingsbury-stone-london/ ### /399-london ### /new-homes/ ### /hadston-london/ ### /new-homes/ ### /the-whiteley-london ### /ae ### / ### /new-homes/ ### /the-chocolate-quarter-london ### / ### / ### /the-silk-district-london ### / ### /new-homes/ ### /battersea-power-station-london ### /the-claves-london ### / ### /new-homes/ ### /whitelocke-house-london ### /new-homes/ ### /shoreditch-parkside-london ### /new-homes/ ### /cassia-london ### /new-homes/ ### /purley-development-london ### /new-homes/ ### /developers ### /rer-london/ ### /developers ### /battersea-power-station-london ### / ### /new-homes/ ### /battersea-power-station-london ### /battersea-power-station-london/floorplans ### /battersea-power-station-london ### /developers ### /new-homes/year-finished/ ### /developers ### /new-homes/ ### /battersea-power-station-london ### /battersea-power-station-london ### /new-homes/ ### /the-platform-london ### /nine-elms-park-e-f-gс ### /nine-elms-park-e-f-gс-london ### /the-platform-london ### /new-homes/ ### / ### /wardian-london ### / ### /eastman-village-london ### /hayes-village-london ### / ### /battersea-power-station-london ### /battersea-power-station-london/gallery ### / ### /new-homes/ ### /cambridge-house-london ### /new-homes/station-east-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /new-homes/station-west-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /cambridge-house-london ### /cambrige-house-hotel-and-residences-london ### /new-homes/ ### /140-northolt-road-london ### / ### / ### / ### / ### /goodluck-hope-london ### /new-homes/ ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### / ### / ### /new-homes/barking/ ### /new-homes/harlesden/ ### /new-homes/barking/ ### /new-homes/marylebone/ ### /new-homes/hackney/ ### /high-street-quarter-london ### / ### /high-street-quarter-london ### /new-homes/ ### /wardian-london ### /battersea-power-station-london ### /new-homes/ ### /verdo-kew-bridge-london ### /new-homes/tw8/ ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### /new-homes/ ### /new-homes/page-20 ### / ### /goodluck-hope-london ### / ### /new-homes/canary-wharf/ ### /goodluck-hope-london/floorplans ### /goodluck-hope-london ### /goodluck-hope-london/floorplans ### / ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### / ### /goodluck-hope-london ### / ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### /dylon-riverside-london ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### / ### /battersea-power-station-london ### /park-modern-london ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### /southmere-london ### / ### /new-homes/ ### /new-homes/central-london-area/ ### / ### /park-modern-london ### /neighbourhood/west-london ### /neighbourhood/ ### /new-homes/ ### /new-homes/property-1-bedroom/ ### /new-homes/canary-wharf/ ### /new-homes/canary-wharf/page-2 ### / ### /eastman-village-london ### /hayes-village-london ### / ### /the-silk-district-london ### /1a-st-johns-wood-park-london ### /goodluck-hope-london ### /brigade-court-london ### /wardian-london ### /hendon-waterside-london ### /new-homes/ ### /10-park-drive-london ### /new-homes/options-health-club/; PHPSESSID={PHPSESSID}; utm=; cookie_agree=1; _ym_isad=1''' } response = requests.request("POST", url, headers=headers, data=payload, timeout=30) res_json = response.json() if res_json.get('success'): res = res_json.get('data') json_data = res.get('resultsJSON') station.json = json_data station.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} station {name} get data error:', e) else: lc = time.time() - 3600*24 ONHStation.objects.filter(last_check__lt=lc).update(json=None) def get_onh_district_json(): district = ONHDistrict.objects.filter(json__isnull=True).first() if district: name = district.name city = district.city code = district.code if city.name == "Dubai": base_url = "https://1newhomes.ae/new-homes/" url = f"https://1newhomes.ae/assets/components/msearch2/action-search.php" else: base_url = "https://1newhomes.com/new-homes/" url = f"https://1newhomes.com/assets/components/msearch2/action-search.php" try: req_co = requests.get(base_url, timeout=30) if req_co.status_code == 200: co = req_co.cookies PHPSESSID = co['PHPSESSID'] html_content = req_co.text soup = BeautifulSoup(html_content, 'html.parser') script_tag = soup.find('script', text=lambda t: t and 'mse2Config' in t) script_content = script_tag.string.replace(",\n", ", ").replace(",\r", ", ").replace("\t","").replace(" "," ").replace(" "," ") mse2Config_match = re.search(r'mse2Config\s*=\s*({.*})', script_content, re.DOTALL) key = "30e35bbdbc0d5a572a05f1b709551ada3d2b5e7b" if mse2Config_match: mse2Config = mse2Config_match.group(1) mse2Config_json = json.loads(mse2Config) key = mse2Config_json.get('key') payload = f"district={code}&action=filter&pageId=2923&key={key}" if city.name == "Dubai": headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''linkchain=/ ### /news/a-15-quarter-climb-in-dubai-real-estate-prices ### /developers ### /emaar-properties-dubai/ ### / ### / ### / ### /new-homes/ ### /the-address-jumeirah-resort-and-spa-dubai ### /emaar-properties-dubai/ ### /the-address-jumeirah-resort-and-spa-dubai ### / ### /new-homes/ ### /prescott-real-estate-dubai/ ### /developers ### /new-homes/ ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /new-homes/page-2 ### /berton-dubai ### /berton-dubai ### /riviera-dubai ### /riviera-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai/floorplans ### / ### /the-mansions-dubai ### / ### /w1nner-tower-winner-tower-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/page-100 ### /samana-lake-views-dubai ### / ### /golf-grand1-dubai ### /new-homes/ ### /new-homes/page-100 ### /the-legends-villas-dubai ### / ### /claydon-house-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /golf-grand1-dubai ### /emaar-properties-dubai/ ### /new-homes/ ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-20 ### /new-homes/property-studio/page-30 ### /damac-properties-dubai/ ### /new-homes/property-studio/page-30 ### /merano-tower-dubai ### /sky-hills-residences-2-dubai ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-30 ### /golf-grand1-dubai ### /camelia-villas-dubai ### /new-homes/property-studio/page-30 ### /liv-waterside-dubai ### /new-homes/ ### /elvira-dhe-dubai ### /new-homes/ ### /the-vybe-dubai ### /the-vybe-dubai ### /sky-hills-residences-2-dubai ### /new-homes/ ### /j-one-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /j-one-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /imperial-avenue-dubai ### /new-homes/ ### /damac-hills-utopia-dubai ### /new-homes/apartments/property-studio/ ### /j-one-dubai ### /damac-hills-utopia-dubai ### /al-jawhara-tower-dubai ### /al-jawhara-tower-dubai/gallery ### /new-homes/apartments/property-studio/ ### /new-homes/apartments/property-studio/ ### /moonsa-residences-2-dubai ### /new-homes/apartments/property-studio/ ### /linden-residences-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /j-one-dubai/gallery ### /' ### / ### /new-homes/ ### /mariane-tower-dubai ### /developers ### /samana-developers-dubai/ ### / ### /imperial-avenue-dubai ### / ### /peninsula-dubai ### / ### /damac-lagoons-morocco-2-dubai ### / ### /damac-casa-dubai ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /parkway-dubai ### /new-homes/ ### /parkway-dubai ### /parkway-dubai/gallery ### /parkway-dubai ### /new-homes/apartments/ ### /new-homes/ ### /riviera-dubai ### / ### /parkway-dubai ### / ### / ### / ### /new-homes/ ### /new-homes/station-jumeirah-beach-residence/ ### /alta-real-estate-developments-dubai/ ### /mr-c-residences-jumeirah-dubai ### / ### / ### /city-walk-northline-dubai ### /parkside-hills-dubai ### /new-homes/ ### /peninsula-dubai ### /parkside-hills-dubai ### / ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /beverly-hills-drive-dubai ### /new-homes/ ### /ag-square-dubai ### /new-homes/ ### / ### /new-homes/ ### /mina-dubai ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### / ### /zada-tower-dubai ### /new-homes/ ### /new-homes/ ### /linden-residences-dubai ### /linden-residences-dubai ### /new-homes/ ### /beach-walk-3-dubai ### / ### /harbour-gate-dubai ### /new-homes/ ### /park-gate-residences-dubai ### /new-homes/ ### /merano-tower-dubai ### / ### /vanguard-by-franсk-muller-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-by-address-resorts-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /new-homes/ ### /davinci-tower-dubai ### /new-homes/ ### /mudon-al-ranim-1-dubai ### /new-homes/ ### /kaya-dubai ### /kaya-dubai ### /new-homes/; _ga_SVS4S7PFEJ=GS1.1.1730685094.108.1.1730686116.0.0.0; _ga=GA1.1.415606180.1724083972; _ym_uid=1719929312736223957; _ym_d=1724083975; cookie_agree=1; PHPSESSID={PHPSESSID}; utm=; _ym_isad=1; _ym_visorc=w''' } else: headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''_ga_JCEVRN83E0=GS1.1.1730685999.287.1.1730686001.58.0.0; _ga=GA1.1.65359444.1719929309; _ym_uid=1719929310452810339; _ym_d=1719929310; _fbp=fb.1.1722030370979.323930974364926; _ga_JCEVRN83E0=deleted; _gcl_au=1.1.962196760.1727727415; linkchain=/ ### /hendon-waterside-london ### / ### /eastman-village-london ### /new-homes/ ### /almax-group-london/ ### /new-homes/year-finished/ ### /1a-st-johns-wood-park-london ### /1a-st-johns-wood-park-london ### /new-homes/?property=1-bedroom&year=Finished ### / ### /new-homes/ ### / ### /new-homes/ ### / ### /developers ### / ### /new-homes/ ### / ### /new-homes/ ### /"https:////1newhomes.com//almax-group-london///" ### /new-homes/ ### / ### /invest ### /first-time-buyers ### /first-time-buyers ### /first-time-buyers ### /the-waterline-london ### /new-homes/options-concierge-service/ ### / ### / ### /luxury-properties ### / ### / ### /developers ### /new-homes/ ### / ### /new-homes/ ### /kingsbury-stone-london/ ### /399-london ### /new-homes/ ### /hadston-london/ ### /new-homes/ ### /the-whiteley-london ### /ae ### / ### /new-homes/ ### /the-chocolate-quarter-london ### / ### / ### /the-silk-district-london ### / ### /new-homes/ ### /battersea-power-station-london ### /the-claves-london ### / ### /new-homes/ ### /whitelocke-house-london ### /new-homes/ ### /shoreditch-parkside-london ### /new-homes/ ### /cassia-london ### /new-homes/ ### /purley-development-london ### /new-homes/ ### /developers ### /rer-london/ ### /developers ### /battersea-power-station-london ### / ### /new-homes/ ### /battersea-power-station-london ### /battersea-power-station-london/floorplans ### /battersea-power-station-london ### /developers ### /new-homes/year-finished/ ### /developers ### /new-homes/ ### /battersea-power-station-london ### /battersea-power-station-london ### /new-homes/ ### /the-platform-london ### /nine-elms-park-e-f-gс ### /nine-elms-park-e-f-gс-london ### /the-platform-london ### /new-homes/ ### / ### /wardian-london ### / ### /eastman-village-london ### /hayes-village-london ### / ### /battersea-power-station-london ### /battersea-power-station-london/gallery ### / ### /new-homes/ ### /cambridge-house-london ### /new-homes/station-east-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /new-homes/station-west-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /cambridge-house-london ### /cambrige-house-hotel-and-residences-london ### /new-homes/ ### /140-northolt-road-london ### / ### / ### / ### / ### /goodluck-hope-london ### /new-homes/ ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### / ### / ### /new-homes/barking/ ### /new-homes/harlesden/ ### /new-homes/barking/ ### /new-homes/marylebone/ ### /new-homes/hackney/ ### /high-street-quarter-london ### / ### /high-street-quarter-london ### /new-homes/ ### /wardian-london ### /battersea-power-station-london ### /new-homes/ ### /verdo-kew-bridge-london ### /new-homes/tw8/ ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### /new-homes/ ### /new-homes/page-20 ### / ### /goodluck-hope-london ### / ### /new-homes/canary-wharf/ ### /goodluck-hope-london/floorplans ### /goodluck-hope-london ### /goodluck-hope-london/floorplans ### / ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### / ### /goodluck-hope-london ### / ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### /dylon-riverside-london ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### / ### /battersea-power-station-london ### /park-modern-london ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### /southmere-london ### / ### /new-homes/ ### /new-homes/central-london-area/ ### / ### /park-modern-london ### /neighbourhood/west-london ### /neighbourhood/ ### /new-homes/ ### /new-homes/property-1-bedroom/ ### /new-homes/canary-wharf/ ### /new-homes/canary-wharf/page-2 ### / ### /eastman-village-london ### /hayes-village-london ### / ### /the-silk-district-london ### /1a-st-johns-wood-park-london ### /goodluck-hope-london ### /brigade-court-london ### /wardian-london ### /hendon-waterside-london ### /new-homes/ ### /10-park-drive-london ### /new-homes/options-health-club/; PHPSESSID={PHPSESSID}; utm=; cookie_agree=1''' } response = requests.request("POST", url, headers=headers, data=payload, timeout=30) res_json = response.json() if res_json.get('success'): res = res_json.get('data') json_data = res.get('resultsJSON') district.json = json_data district.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} district {name} get data error:', e) else: lc = time.time() - 3600*24 ONHDistrict.objects.filter(last_check__lt=lc).update(json=None) def get_onh_postcode_json(): postcode = ONHPostcode.objects.filter(json__isnull=True).first() if postcode: name = postcode.name city = postcode.city code = postcode.code if city.name == "Dubai": base_url = "https://1newhomes.ae/new-homes/" url = f"https://1newhomes.ae/assets/components/msearch2/action-search.php" else: base_url = "https://1newhomes.com/new-homes/" url = f"https://1newhomes.com/assets/components/msearch2/action-search.php" try: req_co = requests.get(base_url, timeout=10) if req_co.status_code == 200: co = req_co.cookies PHPSESSID = co['PHPSESSID'] html_content = req_co.text soup = BeautifulSoup(html_content, 'html.parser') script_tag = soup.find('script', text=lambda t: t and 'mse2Config' in t) script_content = script_tag.string.replace(",\n", ", ").replace(",\r", ", ").replace("\t","").replace(" "," ").replace(" "," ") mse2Config_match = re.search(r'mse2Config\s*=\s*({.*})', script_content, re.DOTALL) key = "30e35bbdbc0d5a572a05f1b709551ada3d2b5e7b" if mse2Config_match: mse2Config = mse2Config_match.group(1) mse2Config_json = json.loads(mse2Config) key = mse2Config_json.get('key') payload = f"postcode={code}&action=filter&pageId=2923&key={key}" if city.name == "Dubai": headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''linkchain=/ ### /news/a-15-quarter-climb-in-dubai-real-estate-prices ### /developers ### /emaar-properties-dubai/ ### / ### / ### / ### /new-homes/ ### /the-address-jumeirah-resort-and-spa-dubai ### /emaar-properties-dubai/ ### /the-address-jumeirah-resort-and-spa-dubai ### / ### /new-homes/ ### /prescott-real-estate-dubai/ ### /developers ### /new-homes/ ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /new-homes/page-2 ### /berton-dubai ### /berton-dubai ### /riviera-dubai ### /riviera-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai/floorplans ### / ### /the-mansions-dubai ### / ### /w1nner-tower-winner-tower-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/page-100 ### /samana-lake-views-dubai ### / ### /golf-grand1-dubai ### /new-homes/ ### /new-homes/page-100 ### /the-legends-villas-dubai ### / ### /claydon-house-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /golf-grand1-dubai ### /emaar-properties-dubai/ ### /new-homes/ ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-20 ### /new-homes/property-studio/page-30 ### /damac-properties-dubai/ ### /new-homes/property-studio/page-30 ### /merano-tower-dubai ### /sky-hills-residences-2-dubai ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-30 ### /golf-grand1-dubai ### /camelia-villas-dubai ### /new-homes/property-studio/page-30 ### /liv-waterside-dubai ### /new-homes/ ### /elvira-dhe-dubai ### /new-homes/ ### /the-vybe-dubai ### /the-vybe-dubai ### /sky-hills-residences-2-dubai ### /new-homes/ ### /j-one-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /j-one-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /imperial-avenue-dubai ### /new-homes/ ### /damac-hills-utopia-dubai ### /new-homes/apartments/property-studio/ ### /j-one-dubai ### /damac-hills-utopia-dubai ### /al-jawhara-tower-dubai ### /al-jawhara-tower-dubai/gallery ### /new-homes/apartments/property-studio/ ### /new-homes/apartments/property-studio/ ### /moonsa-residences-2-dubai ### /new-homes/apartments/property-studio/ ### /linden-residences-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /j-one-dubai/gallery ### /' ### / ### /new-homes/ ### /mariane-tower-dubai ### /developers ### /samana-developers-dubai/ ### / ### /imperial-avenue-dubai ### / ### /peninsula-dubai ### / ### /damac-lagoons-morocco-2-dubai ### / ### /damac-casa-dubai ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /parkway-dubai ### /new-homes/ ### /parkway-dubai ### /parkway-dubai/gallery ### /parkway-dubai ### /new-homes/apartments/ ### /new-homes/ ### /riviera-dubai ### / ### /parkway-dubai ### / ### / ### / ### /new-homes/ ### /new-homes/station-jumeirah-beach-residence/ ### /alta-real-estate-developments-dubai/ ### /mr-c-residences-jumeirah-dubai ### / ### / ### /city-walk-northline-dubai ### /parkside-hills-dubai ### /new-homes/ ### /peninsula-dubai ### /parkside-hills-dubai ### / ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /beverly-hills-drive-dubai ### /new-homes/ ### /ag-square-dubai ### /new-homes/ ### / ### /new-homes/ ### /mina-dubai ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### / ### /zada-tower-dubai ### /new-homes/ ### /new-homes/ ### /linden-residences-dubai ### /linden-residences-dubai ### /new-homes/ ### /beach-walk-3-dubai ### / ### /harbour-gate-dubai ### /new-homes/ ### /park-gate-residences-dubai ### /new-homes/ ### /merano-tower-dubai ### / ### /vanguard-by-franсk-muller-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-by-address-resorts-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /new-homes/ ### /davinci-tower-dubai ### /new-homes/ ### /mudon-al-ranim-1-dubai ### /new-homes/ ### /kaya-dubai ### /kaya-dubai ### /new-homes/; _ga_SVS4S7PFEJ=GS1.1.1730685094.108.1.1730686116.0.0.0; _ga=GA1.1.415606180.1724083972; _ym_uid=1719929312736223957; _ym_d=1724083975; cookie_agree=1; PHPSESSID={PHPSESSID}; utm=; _ym_isad=1; _ym_visorc=w''' } else: headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''_ga_JCEVRN83E0=GS1.1.1730685999.287.1.1730686775.42.0.0; _ga=GA1.1.65359444.1719929309; _ym_uid=1719929310452810339; _ym_d=1719929310; _fbp=fb.1.1722030370979.323930974364926; _ga_JCEVRN83E0=deleted; _gcl_au=1.1.962196760.1727727415; linkchain=/ ### /hendon-waterside-london ### / ### /eastman-village-london ### /new-homes/ ### /almax-group-london/ ### /new-homes/year-finished/ ### /1a-st-johns-wood-park-london ### /1a-st-johns-wood-park-london ### /new-homes/?property=1-bedroom&year=Finished ### / ### /new-homes/ ### / ### /new-homes/ ### / ### /developers ### / ### /new-homes/ ### / ### /new-homes/ ### /"https:////1newhomes.com//almax-group-london///" ### /new-homes/ ### / ### /invest ### /first-time-buyers ### /first-time-buyers ### /first-time-buyers ### /the-waterline-london ### /new-homes/options-concierge-service/ ### / ### / ### /luxury-properties ### / ### / ### /developers ### /new-homes/ ### / ### /new-homes/ ### /kingsbury-stone-london/ ### /399-london ### /new-homes/ ### /hadston-london/ ### /new-homes/ ### /the-whiteley-london ### /ae ### / ### /new-homes/ ### /the-chocolate-quarter-london ### / ### / ### /the-silk-district-london ### / ### /new-homes/ ### /battersea-power-station-london ### /the-claves-london ### / ### /new-homes/ ### /whitelocke-house-london ### /new-homes/ ### /shoreditch-parkside-london ### /new-homes/ ### /cassia-london ### /new-homes/ ### /purley-development-london ### /new-homes/ ### /developers ### /rer-london/ ### /developers ### /battersea-power-station-london ### / ### /new-homes/ ### /battersea-power-station-london ### /battersea-power-station-london/floorplans ### /battersea-power-station-london ### /developers ### /new-homes/year-finished/ ### /developers ### /new-homes/ ### /battersea-power-station-london ### /battersea-power-station-london ### /new-homes/ ### /the-platform-london ### /nine-elms-park-e-f-gс ### /nine-elms-park-e-f-gс-london ### /the-platform-london ### /new-homes/ ### / ### /wardian-london ### / ### /eastman-village-london ### /hayes-village-london ### / ### /battersea-power-station-london ### /battersea-power-station-london/gallery ### / ### /new-homes/ ### /cambridge-house-london ### /new-homes/station-east-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /new-homes/station-west-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /cambridge-house-london ### /cambrige-house-hotel-and-residences-london ### /new-homes/ ### /140-northolt-road-london ### / ### / ### / ### / ### /goodluck-hope-london ### /new-homes/ ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### / ### / ### /new-homes/barking/ ### /new-homes/harlesden/ ### /new-homes/barking/ ### /new-homes/marylebone/ ### /new-homes/hackney/ ### /high-street-quarter-london ### / ### /high-street-quarter-london ### /new-homes/ ### /wardian-london ### /battersea-power-station-london ### /new-homes/ ### /verdo-kew-bridge-london ### /new-homes/tw8/ ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### /new-homes/ ### /new-homes/page-20 ### / ### /goodluck-hope-london ### / ### /new-homes/canary-wharf/ ### /goodluck-hope-london/floorplans ### /goodluck-hope-london ### /goodluck-hope-london/floorplans ### / ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### / ### /goodluck-hope-london ### / ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### /dylon-riverside-london ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### / ### /battersea-power-station-london ### /park-modern-london ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### /southmere-london ### / ### /new-homes/ ### /new-homes/central-london-area/ ### / ### /park-modern-london ### /neighbourhood/west-london ### /neighbourhood/ ### /new-homes/ ### /new-homes/property-1-bedroom/ ### /new-homes/canary-wharf/ ### /new-homes/canary-wharf/page-2 ### / ### /eastman-village-london ### /hayes-village-london ### / ### /the-silk-district-london ### /1a-st-johns-wood-park-london ### /goodluck-hope-london ### /brigade-court-london ### /wardian-london ### /hendon-waterside-london ### /new-homes/ ### /10-park-drive-london ### /new-homes/options-health-club/; PHPSESSID={PHPSESSID}; utm=; cookie_agree=1''' } response = requests.request("POST", url, headers=headers, data=payload, timeout=30) res_json = response.json() if res_json.get('success'): res = res_json.get('data') json_data = res.get('resultsJSON') postcode.json = json_data postcode.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} district {name} get data error:', e) else: lc = time.time() - 3600*24 ONHPostcode.objects.filter(last_check__lt=lc).update(json=None) def get_onh_area_json(): area = ONHArea.objects.filter(json__isnull=True).first() if area: name = area.name city = area.city code = area.code if city.name == "Dubai": base_url = "https://1newhomes.ae/new-homes/" url = f"https://1newhomes.ae/assets/components/msearch2/action-search.php" else: base_url = "https://1newhomes.com/new-homes/" url = f"https://1newhomes.com/assets/components/msearch2/action-search.php" try: req_co = requests.get(base_url, timeout=30) if req_co.status_code == 200: co = req_co.cookies PHPSESSID = co['PHPSESSID'] html_content = req_co.text soup = BeautifulSoup(html_content, 'html.parser') script_tag = soup.find('script', text=lambda t: t and 'mse2Config' in t) script_content = script_tag.string.replace(",\n", ", ").replace(",\r", ", ").replace("\t","").replace(" "," ").replace(" "," ") mse2Config_match = re.search(r'mse2Config\s*=\s*({.*})', script_content, re.DOTALL) key = "30e35bbdbc0d5a572a05f1b709551ada3d2b5e7b" if mse2Config_match: mse2Config = mse2Config_match.group(1) mse2Config_json = json.loads(mse2Config) key = mse2Config_json.get('key') payload = f"area={code}&action=filter&pageId=2923&key={key}" if city.name == "Dubai": headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''linkchain=/ ### /news/a-15-quarter-climb-in-dubai-real-estate-prices ### /developers ### /emaar-properties-dubai/ ### / ### / ### / ### /new-homes/ ### /the-address-jumeirah-resort-and-spa-dubai ### /emaar-properties-dubai/ ### /the-address-jumeirah-resort-and-spa-dubai ### / ### /new-homes/ ### /prescott-real-estate-dubai/ ### /developers ### /new-homes/ ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /new-homes/page-2 ### /berton-dubai ### /berton-dubai ### /riviera-dubai ### /riviera-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai/floorplans ### / ### /the-mansions-dubai ### / ### /w1nner-tower-winner-tower-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/page-100 ### /samana-lake-views-dubai ### / ### /golf-grand1-dubai ### /new-homes/ ### /new-homes/page-100 ### /the-legends-villas-dubai ### / ### /claydon-house-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /golf-grand1-dubai ### /emaar-properties-dubai/ ### /new-homes/ ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-20 ### /new-homes/property-studio/page-30 ### /damac-properties-dubai/ ### /new-homes/property-studio/page-30 ### /merano-tower-dubai ### /sky-hills-residences-2-dubai ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-30 ### /golf-grand1-dubai ### /camelia-villas-dubai ### /new-homes/property-studio/page-30 ### /liv-waterside-dubai ### /new-homes/ ### /elvira-dhe-dubai ### /new-homes/ ### /the-vybe-dubai ### /the-vybe-dubai ### /sky-hills-residences-2-dubai ### /new-homes/ ### /j-one-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /j-one-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /imperial-avenue-dubai ### /new-homes/ ### /damac-hills-utopia-dubai ### /new-homes/apartments/property-studio/ ### /j-one-dubai ### /damac-hills-utopia-dubai ### /al-jawhara-tower-dubai ### /al-jawhara-tower-dubai/gallery ### /new-homes/apartments/property-studio/ ### /new-homes/apartments/property-studio/ ### /moonsa-residences-2-dubai ### /new-homes/apartments/property-studio/ ### /linden-residences-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /j-one-dubai/gallery ### /' ### / ### /new-homes/ ### /mariane-tower-dubai ### /developers ### /samana-developers-dubai/ ### / ### /imperial-avenue-dubai ### / ### /peninsula-dubai ### / ### /damac-lagoons-morocco-2-dubai ### / ### /damac-casa-dubai ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /parkway-dubai ### /new-homes/ ### /parkway-dubai ### /parkway-dubai/gallery ### /parkway-dubai ### /new-homes/apartments/ ### /new-homes/ ### /riviera-dubai ### / ### /parkway-dubai ### / ### / ### / ### /new-homes/ ### /new-homes/area-jumeirah-beach-residence/ ### /alta-real-estate-developments-dubai/ ### /mr-c-residences-jumeirah-dubai ### / ### / ### /city-walk-northline-dubai ### /parkside-hills-dubai ### /new-homes/ ### /peninsula-dubai ### /parkside-hills-dubai ### / ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /beverly-hills-drive-dubai ### /new-homes/ ### /ag-square-dubai ### /new-homes/ ### / ### /new-homes/ ### /mina-dubai ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### / ### /zada-tower-dubai ### /new-homes/ ### /new-homes/ ### /linden-residences-dubai ### /linden-residences-dubai ### /new-homes/ ### /beach-walk-3-dubai ### / ### /harbour-gate-dubai ### /new-homes/ ### /park-gate-residences-dubai ### /new-homes/ ### /merano-tower-dubai ### / ### /vanguard-by-franсk-muller-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-by-address-resorts-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /new-homes/ ### /davinci-tower-dubai ### /new-homes/ ### /mudon-al-ranim-1-dubai ### /new-homes/ ### /kaya-dubai ### /kaya-dubai ### /new-homes/; _ga_SVS4S7PFEJ=GS1.1.1730685094.108.1.1730685108.0.0.0; _ga=GA1.1.415606180.1724083972; _ym_uid=1719929312736223957; _ym_d=1724083975; cookie_agree=1; PHPSESSID={PHPSESSID}; utm=; _ym_isad=1; _ym_visorc=w''' } else: headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''_ga_JCEVRN83E0=GS1.1.1730680201.285.1.1730681307.44.0.0; _ga=GA1.1.65359444.1719929309; _ym_uid=1719929310452810339; _ym_d=1719929310; _fbp=fb.1.1722030370979.323930974364926; _ga_JCEVRN83E0=deleted; _gcl_au=1.1.962196760.1727727415; linkchain=/ ### /hendon-waterside-london ### / ### /eastman-village-london ### /new-homes/ ### /almax-group-london/ ### /new-homes/year-finished/ ### /1a-st-johns-wood-park-london ### /1a-st-johns-wood-park-london ### /new-homes/?property=1-bedroom&year=Finished ### / ### /new-homes/ ### / ### /new-homes/ ### / ### /developers ### / ### /new-homes/ ### / ### /new-homes/ ### /"https:////1newhomes.com//almax-group-london///" ### /new-homes/ ### / ### /invest ### /first-time-buyers ### /first-time-buyers ### /first-time-buyers ### /the-waterline-london ### /new-homes/options-concierge-service/ ### / ### / ### /luxury-properties ### / ### / ### /developers ### /new-homes/ ### / ### /new-homes/ ### /kingsbury-stone-london/ ### /399-london ### /new-homes/ ### /hadston-london/ ### /new-homes/ ### /the-whiteley-london ### /ae ### / ### /new-homes/ ### /the-chocolate-quarter-london ### / ### / ### /the-silk-district-london ### / ### /new-homes/ ### /battersea-power-area-london ### /the-claves-london ### / ### /new-homes/ ### /whitelocke-house-london ### /new-homes/ ### /shoreditch-parkside-london ### /new-homes/ ### /cassia-london ### /new-homes/ ### /purley-development-london ### /new-homes/ ### /developers ### /rer-london/ ### /developers ### /battersea-power-area-london ### / ### /new-homes/ ### /battersea-power-area-london ### /battersea-power-area-london/floorplans ### /battersea-power-area-london ### /developers ### /new-homes/year-finished/ ### /developers ### /new-homes/ ### /battersea-power-area-london ### /battersea-power-area-london ### /new-homes/ ### /the-platform-london ### /nine-elms-park-e-f-gс ### /nine-elms-park-e-f-gс-london ### /the-platform-london ### /new-homes/ ### / ### /wardian-london ### / ### /eastman-village-london ### /hayes-village-london ### / ### /battersea-power-area-london ### /battersea-power-area-london/gallery ### / ### /new-homes/ ### /cambridge-house-london ### /new-homes/area-east-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /new-homes/area-west-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /cambridge-house-london ### /cambrige-house-hotel-and-residences-london ### /new-homes/ ### /140-northolt-road-london ### / ### / ### / ### / ### /goodluck-hope-london ### /new-homes/ ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### / ### / ### /new-homes/barking/ ### /new-homes/harlesden/ ### /new-homes/barking/ ### /new-homes/marylebone/ ### /new-homes/hackney/ ### /high-street-quarter-london ### / ### /high-street-quarter-london ### /new-homes/ ### /wardian-london ### /battersea-power-area-london ### /new-homes/ ### /verdo-kew-bridge-london ### /new-homes/tw8/ ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### /new-homes/ ### /new-homes/page-20 ### / ### /goodluck-hope-london ### / ### /new-homes/canary-wharf/ ### /goodluck-hope-london/floorplans ### /goodluck-hope-london ### /goodluck-hope-london/floorplans ### / ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### / ### /goodluck-hope-london ### / ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### /dylon-riverside-london ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### / ### /battersea-power-area-london ### /park-modern-london ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### /southmere-london ### / ### /new-homes/ ### /new-homes/central-london-area/ ### / ### /park-modern-london ### /neighbourhood/west-london ### /neighbourhood/ ### /new-homes/ ### /new-homes/property-1-bedroom/ ### /new-homes/canary-wharf/ ### /new-homes/canary-wharf/page-2 ### / ### /eastman-village-london ### /hayes-village-london ### / ### /the-silk-district-london ### /1a-st-johns-wood-park-london ### /goodluck-hope-london ### /brigade-court-london ### /wardian-london ### /hendon-waterside-london ### /new-homes/ ### /10-park-drive-london ### /new-homes/options-health-club/; PHPSESSID={PHPSESSID}; utm=; cookie_agree=1; _ym_isad=1''' } response = requests.request("POST", url, headers=headers, data=payload, timeout=30) res_json = response.json() print(res_json) if res_json.get('success'): res = res_json.get('data') json_data = res.get('resultsJSON') area.json = json_data area.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} area {name} get data error:', e) else: lc = time.time() - 3600*24 ONHArea.objects.filter(last_check__lt=lc).update(json=None) def get_onh_zone_json(): zone = ONHZone.objects.filter(json__isnull=True).first() if zone: name = zone.name city = zone.city code = zone.code if city.name == "Dubai": base_url = "https://1newhomes.ae/new-homes/" url = f"https://1newhomes.ae/assets/components/msearch2/action-search.php" else: base_url = "https://1newhomes.com/new-homes/" url = f"https://1newhomes.com/assets/components/msearch2/action-search.php" try: req_co = requests.get(base_url, timeout=30) if req_co.status_code == 200: co = req_co.cookies PHPSESSID = co['PHPSESSID'] html_content = req_co.text soup = BeautifulSoup(html_content, 'html.parser') script_tag = soup.find('script', text=lambda t: t and 'mse2Config' in t) script_content = script_tag.string.replace(",\n", ", ").replace(",\r", ", ").replace("\t","").replace(" "," ").replace(" "," ") mse2Config_match = re.search(r'mse2Config\s*=\s*({.*})', script_content, re.DOTALL) key = "30e35bbdbc0d5a572a05f1b709551ada3d2b5e7b" if mse2Config_match: mse2Config = mse2Config_match.group(1) mse2Config_json = json.loads(mse2Config) key = mse2Config_json.get('key') payload = f"zone={code}&action=filter&pageId=2923&key={key}" if city.name == "Dubai": headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''linkchain=/ ### /news/a-15-quarter-climb-in-dubai-real-estate-prices ### /developers ### /emaar-properties-dubai/ ### / ### / ### / ### /new-homes/ ### /the-address-jumeirah-resort-and-spa-dubai ### /emaar-properties-dubai/ ### /the-address-jumeirah-resort-and-spa-dubai ### / ### /new-homes/ ### /prescott-real-estate-dubai/ ### /developers ### /new-homes/ ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /new-homes/page-2 ### /berton-dubai ### /berton-dubai ### /riviera-dubai ### /riviera-dubai/gallery ### /berton-dubai ### /berton-dubai/gallery ### /berton-dubai/floorplans ### / ### /the-mansions-dubai ### / ### /w1nner-tower-winner-tower-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/page-100 ### /samana-lake-views-dubai ### / ### /golf-grand1-dubai ### /new-homes/ ### /new-homes/page-100 ### /the-legends-villas-dubai ### / ### /claydon-house-dubai ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /golf-grand1-dubai ### /emaar-properties-dubai/ ### /new-homes/ ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-20 ### /new-homes/property-studio/page-30 ### /damac-properties-dubai/ ### /new-homes/property-studio/page-30 ### /merano-tower-dubai ### /sky-hills-residences-2-dubai ### /golf-grand1-dubai ### /peninsula-dubai ### /new-homes/property-studio/page-30 ### /golf-grand1-dubai ### /camelia-villas-dubai ### /new-homes/property-studio/page-30 ### /liv-waterside-dubai ### /new-homes/ ### /elvira-dhe-dubai ### /new-homes/ ### /the-vybe-dubai ### /the-vybe-dubai ### /sky-hills-residences-2-dubai ### /new-homes/ ### /j-one-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /j-one-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /imperial-avenue-dubai ### /new-homes/ ### /damac-hills-utopia-dubai ### /new-homes/apartments/property-studio/ ### /j-one-dubai ### /damac-hills-utopia-dubai ### /al-jawhara-tower-dubai ### /al-jawhara-tower-dubai/gallery ### /new-homes/apartments/property-studio/ ### /new-homes/apartments/property-studio/ ### /moonsa-residences-2-dubai ### /new-homes/apartments/property-studio/ ### /linden-residences-dubai ### /j-one-dubai/floorplans ### /j-one-dubai ### /j-one-dubai/gallery ### /' ### / ### /new-homes/ ### /mariane-tower-dubai ### /developers ### /samana-developers-dubai/ ### / ### /imperial-avenue-dubai ### / ### /peninsula-dubai ### / ### /damac-lagoons-morocco-2-dubai ### / ### /damac-casa-dubai ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /new-homes/ ### /park-avenue-dubai ### /new-homes/ ### /parkway-dubai ### /new-homes/ ### /parkway-dubai ### /parkway-dubai/gallery ### /parkway-dubai ### /new-homes/apartments/ ### /new-homes/ ### /riviera-dubai ### / ### /parkway-dubai ### / ### / ### / ### /new-homes/ ### /new-homes/zone-jumeirah-beach-residence/ ### /alta-real-estate-developments-dubai/ ### /mr-c-residences-jumeirah-dubai ### / ### / ### /city-walk-northline-dubai ### /parkside-hills-dubai ### /new-homes/ ### /peninsula-dubai ### /parkside-hills-dubai ### / ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /new-homes/ ### /beverly-hills-drive-dubai ### /new-homes/ ### /ag-square-dubai ### /new-homes/ ### / ### /new-homes/ ### /mina-dubai ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### /new-homes/ready-to-move/ ### / ### /zada-tower-dubai ### /new-homes/ ### /new-homes/ ### /linden-residences-dubai ### /linden-residences-dubai ### /new-homes/ ### /beach-walk-3-dubai ### / ### /harbour-gate-dubai ### /new-homes/ ### /park-gate-residences-dubai ### /new-homes/ ### /merano-tower-dubai ### / ### /vanguard-by-franсk-muller-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-by-address-resorts-dubai ### /new-homes/ ### /bayview-by-address-dubai ### /new-homes/ ### /bayview-dubai ### /new-homes/ ### /imperial-avenue-dubai ### /new-homes/ ### /davinci-tower-dubai ### /new-homes/ ### /mudon-al-ranim-1-dubai ### /new-homes/ ### /kaya-dubai ### /kaya-dubai ### /new-homes/; _ga_SVS4S7PFEJ=GS1.1.1730685094.108.1.1730685108.0.0.0; _ga=GA1.1.415606180.1724083972; _ym_uid=1719929312736223957; _ym_d=1724083975; cookie_agree=1; PHPSESSID={PHPSESSID}; utm=; _ym_isad=1; _ym_visorc=w''' } else: headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': f'''_ga_JCEVRN83E0=GS1.1.1730680201.285.1.1730681307.44.0.0; _ga=GA1.1.65359444.1719929309; _ym_uid=1719929310452810339; _ym_d=1719929310; _fbp=fb.1.1722030370979.323930974364926; _ga_JCEVRN83E0=deleted; _gcl_au=1.1.962196760.1727727415; linkchain=/ ### /hendon-waterside-london ### / ### /eastman-village-london ### /new-homes/ ### /almax-group-london/ ### /new-homes/year-finished/ ### /1a-st-johns-wood-park-london ### /1a-st-johns-wood-park-london ### /new-homes/?property=1-bedroom&year=Finished ### / ### /new-homes/ ### / ### /new-homes/ ### / ### /developers ### / ### /new-homes/ ### / ### /new-homes/ ### /"https:////1newhomes.com//almax-group-london///" ### /new-homes/ ### / ### /invest ### /first-time-buyers ### /first-time-buyers ### /first-time-buyers ### /the-waterline-london ### /new-homes/options-concierge-service/ ### / ### / ### /luxury-properties ### / ### / ### /developers ### /new-homes/ ### / ### /new-homes/ ### /kingsbury-stone-london/ ### /399-london ### /new-homes/ ### /hadston-london/ ### /new-homes/ ### /the-whiteley-london ### /ae ### / ### /new-homes/ ### /the-chocolate-quarter-london ### / ### / ### /the-silk-district-london ### / ### /new-homes/ ### /battersea-power-zone-london ### /the-claves-london ### / ### /new-homes/ ### /whitelocke-house-london ### /new-homes/ ### /shoreditch-parkside-london ### /new-homes/ ### /cassia-london ### /new-homes/ ### /purley-development-london ### /new-homes/ ### /developers ### /rer-london/ ### /developers ### /battersea-power-zone-london ### / ### /new-homes/ ### /battersea-power-zone-london ### /battersea-power-zone-london/floorplans ### /battersea-power-zone-london ### /developers ### /new-homes/year-finished/ ### /developers ### /new-homes/ ### /battersea-power-zone-london ### /battersea-power-zone-london ### /new-homes/ ### /the-platform-london ### /nine-elms-park-e-f-gс ### /nine-elms-park-e-f-gс-london ### /the-platform-london ### /new-homes/ ### / ### /wardian-london ### / ### /eastman-village-london ### /hayes-village-london ### / ### /battersea-power-zone-london ### /battersea-power-zone-london/gallery ### / ### /new-homes/ ### /cambridge-house-london ### /new-homes/zone-east-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /new-homes/zone-west-croydon/ ### /cambridge-house-london ### /cambridge-house-london ### /cambridge-house-london ### /cambrige-house-hotel-and-residences-london ### /new-homes/ ### /140-northolt-road-london ### / ### / ### / ### / ### /goodluck-hope-london ### /new-homes/ ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### / ### / ### /new-homes/barking/ ### /new-homes/harlesden/ ### /new-homes/barking/ ### /new-homes/marylebone/ ### /new-homes/hackney/ ### /high-street-quarter-london ### / ### /high-street-quarter-london ### /new-homes/ ### /wardian-london ### /battersea-power-zone-london ### /new-homes/ ### /verdo-kew-bridge-london ### /new-homes/tw8/ ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### /new-homes/ ### /new-homes/page-20 ### / ### /goodluck-hope-london ### / ### /new-homes/canary-wharf/ ### /goodluck-hope-london/floorplans ### /goodluck-hope-london ### /goodluck-hope-london/floorplans ### / ### /new-homes/ ### /the-lucan-autograph-collection-residences-london ### /the-lucan-autograph-collection-residences-london/gallery ### / ### /goodluck-hope-london ### / ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### /dylon-riverside-london ### /new-homes/ ### /bermondsey-heights-london ### /new-homes/ ### / ### /battersea-power-zone-london ### /park-modern-london ### /new-homes/ ### / ### /new-homes/ ### /new-homes/ ### /southmere-london ### / ### /new-homes/ ### /new-homes/central-london-zone/ ### / ### /park-modern-london ### /neighbourhood/west-london ### /neighbourhood/ ### /new-homes/ ### /new-homes/property-1-bedroom/ ### /new-homes/canary-wharf/ ### /new-homes/canary-wharf/page-2 ### / ### /eastman-village-london ### /hayes-village-london ### / ### /the-silk-district-london ### /1a-st-johns-wood-park-london ### /goodluck-hope-london ### /brigade-court-london ### /wardian-london ### /hendon-waterside-london ### /new-homes/ ### /10-park-drive-london ### /new-homes/options-health-club/; PHPSESSID={PHPSESSID}; utm=; cookie_agree=1; _ym_isad=1''' } response = requests.request("POST", url, headers=headers, data=payload, timeout=30) res_json = response.json() if res_json.get('success'): res = res_json.get('data') json_data = res.get('resultsJSON') zone.json = json_data zone.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} zone {name} get data error:', e) else: lc = time.time() - 3600*24 ONHZone.objects.filter(last_check__lt=lc).update(json=None) def set_development_station(): lc = time.time() - 3600*24 station = ONHStation.objects.filter(Q(is_check=False) | Q(last_check__lt=lc), json__isnull=False).first() if station: data = station.json country = station.country developments = data.get('locations') for dev in developments: title = dev.get('pagetitle') title = html.unescape(title) slug = slugify(title, allow_unicode=True) try: development = Development.objects.get(slug=slug, country=country) development.onh_station.add(station) except: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {title} not found') station.is_check = True station.last_check = time.time() station.save() def set_development_district(): lc = time.time() - 3600*24 district = ONHDistrict.objects.filter(Q(is_check=False) | Q(last_check__lt=lc), json__isnull=False).first() if district: data = district.json developments = data.get('locations') country = district.country for dev in developments: title = dev.get('pagetitle') title = html.unescape(title) slug = slugify(title, allow_unicode=True) try: development = Development.objects.get(slug=slug, country=country) development.onh_district.add(district) except: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {title} not found') district.is_check = True district.last_check = time.time() district.save() def set_development_postcode(): lc = time.time() - 3600*24 postcode = ONHPostcode.objects.filter(Q(is_check=False) | Q(last_check__lt=lc), json__isnull=False).first() if postcode: data = postcode.json developments = data.get('locations') country = postcode.country for dev in developments: title = dev.get('pagetitle') title = html.unescape(title) slug = slugify(title, allow_unicode=True) try: development = Development.objects.get(slug=slug, country=country) development.onh_postcode.add(postcode) except: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {title} not found') postcode.is_check = True postcode.last_check = time.time() postcode.save() def set_development_area(): lc = time.time() - 3600*24 area = ONHArea.objects.filter(Q(is_check=False) | Q(last_check__lt=lc), json__isnull=False).first() if area: data = area.json developments = data.get('locations') country = area.country for dev in developments: title = dev.get('pagetitle') title = html.unescape(title) slug = slugify(title, allow_unicode=True) try: development = Development.objects.get(slug=slug, country=country) development.onh_area.add(area) except: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {title} not found') area.is_check = True area.last_check = time.time() area.save() def set_development_zone(): lc = time.time() - 3600*24 zone = ONHZone.objects.filter(Q(is_check=False) | Q(last_check__lt=lc), json__isnull=False).first() if zone: data = zone.json developments = data.get('locations') country = zone.country for dev in developments: title = dev.get('pagetitle') title = html.unescape(title) slug = slugify(title, allow_unicode=True) try: development = Development.objects.get(slug=slug, country=country) development.onh_zone.add(zone) except: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {title} not found') zone.is_check = True zone.last_check = time.time() zone.save() def create_filter_seo_objects(): london_payment_options = [] dubai_payment_options = [] london_bedrooms = [] dubai_bedrooms = [] london_features = [] dubai_features = [] london_complitation = [] dubai_complitation = [] for city in City.objects.all(): FilterSeoText.objects.get_or_create(country=city.country, city=city, first_filter=None, second_filter=None) bedrooms = Flat.objects.filter(development__city=city).values_list('bedrooms_num', flat=True).distinct() for bedroom in bedrooms: FilterSeoText.objects.get_or_create(country=city.country, city=city, first_filter=None, second_filter_type="bedroom", second_filter=f'{bedroom}') features = Feature.objects.filter(development_features__development__city=city).values_list('name', flat=True).distinct() for feature in features: FilterSeoText.objects.get_or_create(country=city.country, city=city, first_filter=None, second_filter_type="feature", second_filter=f'{feature}') for station in ONHStation.objects.all(): FilterSeoText.objects.get_or_create(country=station.country, city=station.city, first_filter=station.name, first_filter_type="station", first_filter_prefix="around", second_filter=None) bedrooms = Flat.objects.filter(development__onh_station=station).values_list('bedrooms_num', flat=True).distinct() for bedroom in bedrooms: FilterSeoText.objects.get_or_create(country=station.country, city=station.city, first_filter=station.name, first_filter_type="station", first_filter_prefix="around", second_filter_type="bedroom", second_filter=f'{bedroom}') features = Feature.objects.filter(development_features__development__onh_station=station).values_list('name', flat=True).distinct() for feature in features: FilterSeoText.objects.get_or_create(country=station.country, city=station.city, first_filter=station.name, first_filter_type="station", first_filter_prefix="around", second_filter_type="feature", second_filter=f'{feature}') for district in ONHDistrict.objects.all(): FilterSeoText.objects.get_or_create(country=district.country, city=district.city, first_filter=district.name, first_filter_type="district", first_filter_prefix="in", second_filter=None) bedrooms = Flat.objects.filter(development__onh_district=district).values_list('bedrooms_num', flat=True).distinct() for bedroom in bedrooms: FilterSeoText.objects.get_or_create(country=district.country, city=district.city, first_filter=district.name, first_filter_type="district", first_filter_prefix="in", second_filter_type="bedroom", second_filter=f'{bedroom}') features = Feature.objects.filter(development_features__development__onh_district=district).values_list('name', flat=True).distinct() for feature in features: FilterSeoText.objects.get_or_create(country=district.country, city=district.city, first_filter=district.name, first_filter_type="district", first_filter_prefix="in", second_filter_type="feature", second_filter=f'{feature}') for postcode in ONHPostcode.objects.all(): FilterSeoText.objects.get_or_create(country=postcode.country, city=postcode.city, first_filter=postcode.name, first_filter_type="postcode", first_filter_prefix="in", second_filter=None) bedrooms = Flat.objects.filter(development__onh_postcode=postcode).values_list('bedrooms_num', flat=True).distinct() for bedroom in bedrooms: FilterSeoText.objects.get_or_create(country=postcode.country, city=postcode.city, first_filter=postcode.name, first_filter_type="postcode", first_filter_prefix="in", second_filter_type="bedroom", second_filter=f'{bedroom}') features = Feature.objects.filter(development_features__development__onh_postcode=postcode).values_list('name', flat=True).distinct() for feature in features: FilterSeoText.objects.get_or_create(country=postcode.country, city=postcode.city, first_filter=postcode.name, first_filter_type="postcode", first_filter_prefix="in", second_filter_type="feature", second_filter=f'{feature}') for area in ONHArea.objects.all(): FilterSeoText.objects.get_or_create(country=area.country, city=area.city, first_filter=area.name, first_filter_type="area", first_filter_prefix="in", second_filter=None) bedrooms = Flat.objects.filter(development__onh_area=area).values_list('bedrooms_num', flat=True).distinct() for bedroom in bedrooms: FilterSeoText.objects.get_or_create(country=area.country, city=area.city, first_filter=area.name, first_filter_type="area", first_filter_prefix="in", second_filter_type="bedroom", second_filter=f'{bedroom}') features = Feature.objects.filter(development_features__development__onh_area=area).values_list('name', flat=True).distinct() for feature in features: FilterSeoText.objects.get_or_create(country=area.country, city=area.city, first_filter=area.name, first_filter_type="area", first_filter_prefix="in", second_filter_type="feature", second_filter=f'{feature}') for zone in ONHZone.objects.all(): FilterSeoText.objects.get_or_create(country=zone.country, city=zone.city, first_filter=zone.name, first_filter_type="zone", first_filter_prefix="in", second_filter=None) bedrooms = Flat.objects.filter(development__onh_zone=zone).values_list('bedrooms_num', flat=True).distinct() for bedroom in bedrooms: FilterSeoText.objects.get_or_create(country=zone.country, city=zone.city, first_filter=zone.name, first_filter_type="zone", first_filter_prefix="in", second_filter_type="bedroom", second_filter=f'{bedroom}') features = Feature.objects.filter(development_features__development__onh_zone=zone).values_list('name', flat=True).distinct() for feature in features: FilterSeoText.objects.get_or_create(country=zone.country, city=zone.city, first_filter=zone.name, first_filter_type="zone", first_filter_prefix="in", second_filter_type="feature", second_filter=f'{feature}') def generate_seo_text(area, neighborhood, features, price, developments, language="english", max_tokens=1800, model="gpt-4o-mini", available=None, completion_date=None, ready_to_move=None): base_prompt = f""" You are a professional copywriter creating a detailed and engaging text about new build properties and apartments for sale in {area}. This text is intended for a property website targeting potential buyers, including families, first-time homeowners, and investors. The tone should be friendly, informative, and persuasive, with clear headings and bullet points for lists where appropriate. The text should be in {language}. The text should be in HTML format, without , , and tags, and for heading tags, use

-

tags. Requirements Tone and Style: Conversational, engaging, and persuasive, designed to inspire action while providing useful information. Include rhetorical questions and relatable scenarios to make the text human-like and engaging. Use phrases that emphasize the uniqueness of the properties and apartments and create a sense of urgency. Structure: Captivating Introduction: Why buying a property or apartment in a new development in {area} is the best decision for families, professionals, or investors looking for long-term value and comfort. Neighborhoods: Highlight key areas ({neighborhood}) with details about excellent transport links, local schools, parks, and amenities that make these areas desirable. Benefits of New Builds: Discuss the advantages of buying new builds, including features like {features} such as modern architecture, energy efficiency, and smart home capabilities. Modern Amenities: Showcase the benefits of apartments in new developments, emphasizing features like {features}, offering modern, comfortable living with safety and convenience. """ if available: base_prompt += f"""Availability: Mention property and apartment availability status such as {available}, emphasizing that the best options are selling fast. Use phrases like “Limited Availability” to create urgency.""" if completion_date: base_prompt += f"""Completion Date: Provide expected or actual completion dates {completion_date}, helping buyers plan their next steps with confidence.""" if ready_to_move: base_prompt += f"""Ready to Move: Highlight "Ready to Move" properties {ready_to_move} for buyers eager to settle in quickly and enjoy the convenience of moving into a finished home.""" base_prompt += f""" Pricing Overview: Include price ranges {price}, emphasizing flexibility for different budgets and how these properties offer long-term value and growth potential. Featured Developments: Introduce standout projects such as {developments}, detailing their unique features and explaining why they represent a solid investment. A Future-Proof Investment: Discuss the potential returns of owning a property in {area}, including price trends and the advantages of investing in high-demand neighborhoods. Dynamic Call to Action: End with an engaging call to action, encouraging readers to explore listings, schedule a visit, or secure their dream property today. Key Guidelines: Text length: 600-800 words. Maintain a natural, engaging tone that feels human-written, while creating urgency to prompt action. Ensure the text is fluent, easily understandable, and highlights key insights for potential clients. Sales-focused: Every section should address customer needs and highlight solutions, making the text both informative and persuasive. Avoid repetitive or generic phrasing, and keep the content fresh, dynamic, and actionable. """ try: client = OpenAI( api_key=openai_key, ) response = client.chat.completions.create( model = model, messages = [{"role": "system", "content": ( "You are an experienced real estate consultant and content optimization specialist. " "Your role is to analyze, refine, and enhance user prompts to maximize their effectiveness. " "You focus on creating SEO-friendly, engaging, and persuasive text for real estate listings. " "Always ensure the tone aligns with the target audience (families, first-time buyers, investors) " "and the structure is clear and sales-oriented. Provide actionable feedback and suggest improvements to meet the user's goals." )}, {"role": "user", "content": base_prompt}], temperature = 0.7, max_tokens=max_tokens, n=1, stop=None ) # Extract text from response generated_text = response.choices[0].message.content generated_text = generated_text.replace("```html", "").replace("```", "") return generated_text except Exception as e: print(f"Error generating content: {e}") return None def add_english_seo_text(): fst = FilterSeoText.objects.filter(translates__isnull=True, city__isnull=False)[:10] languge = Language.objects.get(code='en') for f in fst: country = f.country area = f"in {country.name}" currency = country.default_currency if currency is None: if country.code == "GB": currency = Currency.objects.get(code="GBP") else: currency = Currency.objects.get(code="AED") neighborhood = "London" if country.code == "GB" else "Dubai" city = f.city first_filter = f.first_filter first_filter_type = f.first_filter_type first_filter_prefix = f.first_filter_prefix second_filter = f.second_filter second_filter_type = f.second_filter_type second_filter_prefix = f.second_filter_prefix df = Q(country=country) if city: neighborhood = ", ".join([d.name for d in city.onh_districts.all().order_by("?")[:2]]) area = f"in {city.name} city" df &= Q(city=city) if first_filter: area = f"{first_filter_prefix} {first_filter} in {city.name} city" if first_filter_type == "station": df &= Q(onh_station__name=first_filter) elif first_filter_type == "district": neighborhood = ", ".join([d.name for d in city.onh_area.all().order_by("?")[:2]]) df &= Q(onh_district__name=first_filter) elif first_filter_type == "postcode": neighborhood = ", ".join([d.name for d in city.onh_area.all().order_by("?")[:2]]) df &= Q(onh_postcode__name=first_filter) elif first_filter_type == "area": neighborhood = ", ".join([d.name for d in city.onh_zone.all().order_by("?")[:2]]) df &= Q(onh_area__name=first_filter) elif first_filter_type == "zone": neighborhood = ", ".join([d.name for d in city.onh_station.all().order_by("?")[:2]]) df &= Q(onh_zone__name=first_filter) if second_filter: if second_filter_type == "bedroom": area = f"{second_filter_prefix} {flat_bedroom_type(second_filter)} flats {area}" df &= Q(flats__bedrooms_num=second_filter) elif second_filter_type == "feature": area = f"{second_filter_prefix} {second_filter} {area}" df &= Q(key_features__feature__name=second_filter) developments = Development.objects.filter(df).distinct() features = ", ".join([f.name for f in Feature.objects.filter(development_features__development__in=developments).distinct()[:3]]) try: min_prices = min(Flat.objects.filter(development__in=developments,base_price__gt=0).annotate(ck_price=Min('base_price')).values_list('ck_price', flat=True)) except: min_prices = 0 try: max_prices = max(Flat.objects.filter(development__in=developments,base_price__gt=0).annotate(ck_price=Max('base_price')).values_list('ck_price', flat=True)) except: max_prices = "50000000" price = f"{min_prices} - {max_prices} {currency.code}" developments = ", ".join([d.title for d in developments[:2]]) text = generate_seo_text(area, neighborhood, features, price, developments) if text: f.translates.create(language=languge, text=text) def text_translate(text,language,model="gpt-4o-mini"): prompt = f""" translate the following text to {language} language without changing the meaning of the text in html format and without html,body and head tags: {text} """ try: client = OpenAI( api_key=openai_key, ) chat = client.chat.completions.create( model = model, messages = [{"role": "system", "content": "You are an experienced real estate consultant with deep market knowledge."}, {"role": "user", "content": prompt}], temperature = 0.7 ) generated_text = chat.choices[0].message.content generated_text = generated_text.replace("```html", "").replace("```", "") return {"error": False, "content":generated_text, "error_txt":None} except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in text translate error : ', e) return {"error": True, "content":None, "error_txt":f"{e}"} def translate_seo_text(): languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") fst = FilterSeoText.objects.annotate(tr_num=Count('translates')).filter(tr_num__lt=languages.count()+1).first() if fst: try: text = fst.translates.filter(language=base_language).first().text for l in languages: if not fst.translates.filter(language=l).exists(): res = text_translate(text,l.name) if res["error"]: print(f"error on {fst.id} translate to {l.code} language") else: fst.translates.create(language=l, text=res["content"]) except: pass def create_development_faq(): fa = Language.objects.get(code="fa") en = Language.objects.get(code="en") ru = Language.objects.get(code="ru") ar = Language.objects.get(code="ar") cat1, _ = FaqCategory.objects.get_or_create(title="Property Details") cat_fa, _ = FaqCategoryTranslate.objects.get_or_create(language=fa, category=cat1, title="اطلاعات ملک") cat_en, _ = FaqCategoryTranslate.objects.get_or_create(language=en, category=cat1, title="Property Details") cat_ru, _ = FaqCategoryTranslate.objects.get_or_create(language=ru, category=cat1, title="Общая информация") cat_ar, _ = FaqCategoryTranslate.objects.get_or_create(language=ar, category=cat1, title="تفاصيل الخاصية") cat2, _ = FaqCategory.objects.get_or_create(title="Prices") cat_fa, _ = FaqCategoryTranslate.objects.get_or_create(language=fa, category=cat2, title="قیمت") cat_en, _ = FaqCategoryTranslate.objects.get_or_create(language=en, category=cat2, title="Prices") cat_ru, _ = FaqCategoryTranslate.objects.get_or_create(language=ru, category=cat2, title="Цены") cat_ar, _ = FaqCategoryTranslate.objects.get_or_create(language=ar, category=cat2, title="الأسعار") cat3, _ = FaqCategory.objects.get_or_create(title="Amenities and Features") cat_fa, _ = FaqCategoryTranslate.objects.get_or_create(language=fa, category=cat3, title="تسهیلات و ویژگی‌ها") cat_en, _ = FaqCategoryTranslate.objects.get_or_create(language=en, category=cat3, title="Amenities and Features") cat_ru, _ = FaqCategoryTranslate.objects.get_or_create(language=ru, category=cat3, title="Удобства и особенности") cat_ar, _ = FaqCategoryTranslate.objects.get_or_create(language=ar, category=cat3, title="وسائل الراحة والميزات") cat4, _ = FaqCategory.objects.get_or_create(title="Location and Accessibility") cat_fa, _ = FaqCategoryTranslate.objects.get_or_create(language=fa, category=cat4, title="موقعیت مکانی و دسترسی") cat_en, _ = FaqCategoryTranslate.objects.get_or_create(language=en, category=cat4, title="Location and Accessibility") cat_ru, _ = FaqCategoryTranslate.objects.get_or_create(language=ru, category=cat4, title="Расположение и доступность") cat_ar, _ = FaqCategoryTranslate.objects.get_or_create(language=ar, category=cat4, title="الموقع وسهولة الوصول") cat5, _ = FaqCategory.objects.get_or_create(title="Nearby Services") cat_fa, _ = FaqCategoryTranslate.objects.get_or_create(language=fa, category=cat5, title="خدمات اطراف") cat_en, _ = FaqCategoryTranslate.objects.get_or_create(language=en, category=cat5, title="Nearby Services") cat_ru, _ = FaqCategoryTranslate.objects.get_or_create(language=ru, category=cat5, title="Близлежащие услуги") cat_ar, _ = FaqCategoryTranslate.objects.get_or_create(language=ar, category=cat5, title="الخدمات القريبة") cat6, _ = FaqCategory.objects.get_or_create(title="Sustainability") cat_fa, _ = FaqCategoryTranslate.objects.get_or_create(language=fa, category=cat6, title="پایداری") cat_en, _ = FaqCategoryTranslate.objects.get_or_create(language=en, category=cat6, title="Sustainability") cat_ru, _ = FaqCategoryTranslate.objects.get_or_create(language=ru, category=cat6, title="Экологичность") cat_ar, _ = FaqCategoryTranslate.objects.get_or_create(language=ar, category=cat6, title="الاستدامة") developments = Development.objects.annotate(tr_num=Count('faqs')).filter(tr_num__lt=5) if developments.count() == 0: dt = time.time() - 24 * 60 * 60 developments = Development.objects.filter(Q(faqs__isnull=True) | Q(faqs__translates__isnull=True) | Q(faqs__translates__created_time__lt=dt)).distinct() for development in developments[:100]: development_en = development.translates.filter(language=en).first() development_fa = development.translates.filter(language=fa).first() or development_en development_ru = development.translates.filter(language=ru).first() or development_en development_ar = development.translates.filter(language=ar).first() or development_en city_en = development.city.translates.filter(language=en).first() city_fa = development.city.translates.filter(language=fa).first() or city_en city_ru = development.city.translates.filter(language=ru).first() or city_en city_ar = development.city.translates.filter(language=ar).first() or city_en default_currency = development.default_currency.code try: question = f"Who is the developer of {development.title}?" faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"The developer of {development.title} is {development.developer.name}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"توسعه‌دهنده {development.title} کیست؟" faq_t.answer = f"توسعه‌دهنده {development.title} {development.developer.name} است." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Кто является застройщиком {development.title}?" faq_t.answer = f"Застройщиком {development.title} является {development.developer.name}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"من هو المطور لـ {development.title}؟" faq_t.answer = f"المطور لـ {development.title} هو {development.developer.name}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"What is the address of {development.title}?" faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"The address of {development.title} is {development_en.address}, {city_en.name}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"آدرس {development.title} کجاست؟" faq_t.answer = f"آدرس {development.title} {development_fa.address}، {city_fa.name} است." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Какой адрес {development.title}?" faq_t.answer = f"Адрес {development.title} - {development_ru.address}, {city_ru.name}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما هو عنوان {development.title}؟" faq_t.answer = f"عنوان {development.title} هو {development_ar.address}، {city_ar.name}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"How many flats are there in {development.title}?" flats = KeyFeature.objects.filter(development=development, feature__slug="flats").first() if flats: faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"The total number of flats in {development.title} is {flats.value}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"تعداد واحدهای {development.title} چند عدد است؟" faq_t.answer = f"تعداد کل واحدها در {development.title} {flats.value} عدد است." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Сколько квартир в {development.title}?" faq_t.answer = f"Общее количество квартир в {development.title} составляет {flats.value}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"كم عدد الشقق الموجودة في {development.title}؟" faq_t.answer = f"إجمالي عدد للشقق في {development.title} هو {flats.value}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"How many floors does {development.title} have?" floors = KeyFeature.objects.filter(development=development, feature__slug="floors").first() if floors: faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"The total number of floors in {development.title} is {floors.value}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"{development.title} چند طبقه دارد؟" faq_t.answer = f"تعداد کل طبقات در {development.title} {floors.value} طبقه است." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Сколько этажей в {development.title}?" faq_t.answer = f"Общее количество этажей в {development.title} составляет {floors.value}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"كم عدد الطوابق في {development.title}؟" faq_t.answer = f"إجمالي عدد للطوابق في {development.title} هو {floors.value}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"What type of property is {development.title}?" property_type_en = [b.translates.filter(language=en).first() for b in development.building_type.all()] property_type_fa = [b.translates.filter(language=fa).first() for b in development.building_type.all()] property_type_ru = [b.translates.filter(language=ru).first() for b in development.building_type.all()] property_type_ar = [b.translates.filter(language=ar).first() for b in development.building_type.all()] property_type_en = ", ".join([b.name for b in property_type_en]) property_type_fa = ", ".join([b.name for b in property_type_fa]) or property_type_en property_type_ru = ", ".join([b.name for b in property_type_ru]) or property_type_en property_type_ar = ", ".join([b.name for b in property_type_ar]) or property_type_en if property_type_en: faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"{development.title} is {property_type_en} property." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"{development.title} چه نوع ملکی است؟" faq_t.answer = f"{development.title} یک پروژه {property_type_fa} است." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Какой тип недвижимости {development.title}?" faq_t.answer = f"{development.title} - это {property_type_ru}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما نوع العقار {development.title}؟" faq_t.answer = f"{development.title} هو عقار من نوع {property_type_ar}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) # try: # question = f"Is {development.title} a new build or a redevelopment?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"{development.title} is a {Development_Type}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"{development.title} یک ساختمان جدید است یا بازسازی شده؟" # faq_t.answer = f"{development.title} یک پروژه {Development_Type} است." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Это новая постройка или реконструкция?" # faq_t.answer = f"{development.title} является {Development_Type}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FqTranslate.objects.get_or_create(faq=faq, language=ar, question=f"هل {development.title} هو بناء جديد أم إعادة تطوير؟", answer=f"{development.title} هو مشروع {Development_Type}.") # faq_t.question = f"هل {development.title} هو بناء جديد أم إعادة تطوير؟" # faq_t.answer = f"{development.title} هو مشروع {Development_Type}." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) try: question = f"What is the zone number of {development.title}?" zone = development.onh_zone if zone: faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"{development.title} is located in {zone.name}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"شماره منطقه {development.title} چیست؟" faq_t.answer = f"{development.title} در منطقه {zone.name} واقع شده است." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Какой номер зоны у {development.title}?" faq_t.answer = f"{development.title} расположен в зоне {zone.name}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما هو رقم المنطقة لـ {development.title}؟" faq_t.answer = f"يقع {development.title} في المنطقة {zone.name}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"What is the postcode of {development.title}?" postcode = development.location.zip_code if postcode: faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"The postcode of {development.title} is {postcode}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"کدپستی {development.title} چیست؟" faq_t.answer = f"کدپستی {development.title} {postcode} است." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Какой почтовый индекс {development.title}?" faq_t.answer = f"Почтовый индекс {development.title} - {postcode}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما هو الرمز البريدي لـ {development.title}؟" faq_t.answer = f"الرمز البريدي لـ {development.title} هو {postcode}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"When will {development.title} be completed?" completed_date = development.completed_date if completed_date: faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) if development.is_ready_to_move: answer_en = f"{development.title} is completed and ready to move in." answer_fa = f"{development.title} تکمیل شده و آماده جهت سرویس دهی است." answer_ru = f"{development.title} завершен и готов к заселению." answer_ar = f"{development.title} مكتمل ومستعد للخدمة." else: answer_en = f"{development.title} is expected to be completed by {completed_date}." answer_fa = f"پیش‌بینی می‌شود {development.title} تا {completed_date} تکمیل شود." answer_ru = f"Ожидается, что {development.title} будет завершен к {completed_date}." answer_ar = f"من المتوقع أن يتم الانتهاء من {development.title} بحلول {completed_date}." faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = answer_en faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"زمان تکمیل {development.title} کی خواهد بود؟" faq_t.answer = answer_fa faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Когда будет завершен {development.title}?" faq_t.answer = answer_ru faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"متى سيتم الانتهاء من {development.title}?" faq_t.answer = answer_ar faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) # try: # question = f"Where can I find the {development.title} site plan?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat1, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"The {development.title} site plan is available in the {Site Plan} section on our website." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"نقشه سایت {development.title} را کجا می‌توانم پیدا کنم؟" # faq_t.answer = f"نقشه سایت {development.title} در بخش {Site Plan} در وب‌سایت ما در دسترس است." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Где я могу найти план участка {development.title}?" # faq_t.answer = f"План участка {development.title} доступен в разделе {Site Plan} на нашем сайте." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"أين يمكنني العثور على المخطط الموقعي لـ {development.title}؟" # faq_t.answer = f"المخطط الموقعي لـ {development.title} متاح في قسم {Site Plan} على موقعنا الإلكتروني." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) try: question = f"What is the price range of flats in {development.title}?" if development.is_sold_out: answer_en = f"{development.title} is sold out." answer_fa = f"{development.title} تمام شده است." answer_ru = f"{development.title} продан." answer_ar = f"{development.title} مباع." else: min_price = development.flats.all().exclude(base_price=0).aggregate(Min('base_price'))['base_price__min'] if min_price is None: min_price = 0 max_price = development.flats.all().exclude(base_price=0).aggregate(Max('base_price'))['base_price__max'] if max_price is None: max_price = 0 if min_price == 0: min_price = max_price if min_price == 0: answer_en = f"{development.title} is sold out." answer_fa = f"{development.title} تمام شده است." answer_ru = f"{development.title} продан." answer_ar = f"{development.title} مباع." elif min_price == max_price: answer_en = f"The price range for flats in {development.title} is {default_currency}{min_price}." answer_fa = f"محدوده قیمت آپارتمان‌ها در {development.title} {default_currency}{min_price} است." answer_ru = f"Диапазон цен на квартиры в {development.title} составляет {default_currency}{min_price}." answer_ar = f"يتراوح النطاق السعري للشقق في {development.title} هي {default_currency}{min_price}." else: answer_en = f"The price range for flats in {development.title} is from {default_currency}{min_price} to {default_currency}{max_price}." answer_fa = f"محدوده قیمت آپارتمان‌ها در {development.title} از {default_currency}{min_price} تا {default_currency}{max_price} است." answer_ru = f"Диапазон цен на квартиры в {development.title} составляет от {default_currency}{min_price} до {default_currency}{max_price}." answer_ar = f"يتراوح النطاق السعري للشقق في {development.title} هي من {default_currency}{min_price} إلى {default_currency}{max_price}." faq, _ = Faq.objects.get_or_create(development=development, category=cat2, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = answer_en faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"محدوده قیمت آپارتمان‌ها در {development.title} چقدر است؟" faq_t.answer = answer_fa faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Каков диапазон цен на квартиры в {development.title}?" faq_t.answer = answer_ru faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما هو نطاق السعري الشقق في {development.title}؟" faq_t.answer = answer_ar faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) # try: # question = f"Are there additional fees for residents of {development.title}?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat2, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"Yes, residents are required to pay service charges of approximately {default_currency}{Service_Charge} per year." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"آیا هزینه‌های اضافی برای ساکنان {development.title} وجود دارد؟" # faq_t.answer = f"بله، ساکنان موظف به پرداخت هزینه‌های خدمات به مبلغ تقریبی {default_currency}{Service_Charge} در سال هستند." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Есть ли дополнительные сборы для жильцов {development.title}?" # faq_t.answer = f"Да, жильцы должны платить сервисные сборы примерно {default_currency}{Service_Charge} в год." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"هل هناك رسوم إضافية للمقيمين في {development.title}؟" # faq_t.answer = f"نعم، يُطلب من المقيمين دفع رسوم الخدمات بحوالي {default_currency}{Service_Charge} سنويًا." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) try: question = f"Are financing options available for purchasing flats at {development.title}?" faq, _ = Faq.objects.get_or_create(development=development, category=cat2, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"Yes, financing options are available through our partner banks." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"آیا گزینه‌های تأمین مالی برای خرید آپارتمان در {development.title} موجود است؟" faq_t.answer = f"بله، گزینه‌های تأمین مالی از طریق بانک‌های همکار ما در دسترس است." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Доступны ли варианты финансирования для покупки квартир в {development.title}?" faq_t.answer = f"Да, варианты финансирования доступны через наши банки-партнеры." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"هل خيارات التمويل متاحة لشراء شقق في {development.title}؟" faq_t.answer = f"نعم، تتوفر خيارات التمويل من خلال البنوك الشريكة لدينا." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) # try: # question = f"Is there a reservation process for purchasing flats at {development.title}?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat2, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"Yes, the reservation process includes {Reservation_Process}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"آیا فرآیند رزرو برای خرید آپارتمان در {development.title} وجود دارد؟" # faq_t.answer = f"بله، فرآیند رزرو شامل {Reservation_Process} می‌شود." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Есть ли процесс бронирования для покупки квартир в {development.title}?" # faq_t.answer = f"Да, процесс бронирования включает {Reservation_Process}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"هل هناك عملية حجز لشراء شقق في {development.title}؟" # faq_t.answer = f"نعم، تتضمن عملية الحجز {Reservation_Process}." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) # try: # question = f"Are the flats leasehold or freehold?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat2, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"The flats are {Leasehold_Freehold} with a lease term of {Lease_Term} years." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"آپارتمان‌ها اجاره‌ای (Leasehold) هستند یا مالکیت کامل (Freehold)؟" # faq_t.answer = f"آپارتمان‌ها {Leasehold_Freehold} با مدت اجاره {Lease_Term} سال هستند." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Квартиры в аренде (Leasehold) или в полной собственности (Freehold)?" # faq_t.answer = f"Квартиры {Leasehold_Freehold} сроком аренды {Lease_Term} лет." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"هل الشقق مؤجرة (Leasehold) أم مملوكة بالكامل (Freehold)؟" # faq_t.answer = f"الشقق {Leasehold_Freehold} بمدة إيجار {Lease_Term} سنة." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) try: question = f"What facilities are available to residents of {development.title}?" public_facilities = PublicFacility.objects.filter(type_id__in=[17,18,13,10], developments__development=development) facilities = ", ".join([f"{pf.name} {pf.type.name}" for pf in public_facilities[:3]]) if facilities: faq, _ = Faq.objects.get_or_create(development=development, category=cat3, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"Residents can enjoy {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"چه امکاناتی برای ساکنان {development.title} در دسترس است؟" faq_t.answer = f"ساکنان می‌توانند از {facilities} استفاده کنند." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Какие удобства доступны жильцам {development.title}?" faq_t.answer = f"Жильцы могут пользоваться {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما هي المرافق المتاحة لسكان {development.title}؟" faq_t.answer = f"يمكن للمقيمين الاستمتاع بـ {facilities}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"Are parking spaces available at {development.title}?" faq, _ = Faq.objects.get_or_create(development=development, category=cat3, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"Yes, parking spaces are available." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"آیا جای پارک در {development.title} موجود است؟" faq_t.answer = f"بله، جای پارک در دسترس است." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Доступны ли парковочные места в {development.title}?" faq_t.answer = f"Да, парковочные места." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"هل تتوفر أماكن وقوف السيارات في {development.title}؟" faq_t.answer = f"نعم، تتوفر أماكن وقوف السيارات." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) # try: # question = f"Are pets allowed in {development.title}?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat3, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"{Pet_Policy}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"آیا حیوانات خانگی در {development.title} مجاز هستند؟" # faq_t.answer = f"{Pet_Policy}" # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Разрешены ли домашние животные в {development.title}?" # faq_t.answer = f"{Pet_Policy}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"هل يُسمح بالحيوانات الأليفة في {development.title}؟" # faq_t.answer = f"{Pet_Policy}." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) # try: # question = f"Does {development.title} offer storage spaces for residents?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat3, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"Yes, storage spaces are available at {Storage_Details}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"آیا فضاهای ذخیره‌سازی برای ساکنان {development.title} وجود دارد؟" # faq_t.answer = f"بله، فضاهای ذخیره‌سازی در {Storage_Details} موجود است." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Предлагает ли {development.title} места для хранения вещей жильцов?" # faq_t.answer = f"Да, места для хранения доступны в {Storage_Details}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"هل يوفر {development.title} مساحات تخزين للمقيمين؟" # faq_t.answer = f"نعم، تتوفر مساحات التخزين في {Storage_Details}." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) # try: # question = f"Can residents customize their flats before moving in?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat3, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"Yes, customization options include {Customization_Options}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"آیا ساکنان می‌توانند آپارتمان‌های خود را قبل از اسباب‌کشی سفارشی کنند؟" # faq_t.answer = f"بله، گزینه‌های سفارشی‌سازی شامل {Customization_Options} می‌شود." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Могут ли жильцы настроить свои квартиры перед переездом?" # faq_t.answer = f"Да, варианты настройки включают {Customization_Options}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"هل يمكن للمقيمين تخصيص شققهم قبل الانتقال؟" # faq_t.answer = f"نعم، تشمل خيارات التخصيص {Customization_Options}." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) # try: # question = f"Are there premium upgrade packages available?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat3, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"Yes, premium upgrade packages such as {Upgrade_Packages} are available for an additional cost." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"آیا بسته‌های ارتقای ویژه موجود است؟" # faq_t.answer = f"بله، بسته‌های ارتقای ویژه‌ای مانند {Upgrade_Packages} با هزینه اضافی در دسترس هستند." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Доступны ли премиальные пакеты обновления?" # faq_t.answer = f"Да, премиальные пакеты обновления, такие как {Upgrade_Packages}, доступны за дополнительную плату." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"هل هناك حزم ترقية premium متاحة؟" # faq_t.answer = f"نعم، تتوفر حزم الترقية premium مثل {Upgrade_Packages} مقابل تكلفة إضافية." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) try: question = f"How far is {development.title} from central {development.city.name}?" distance = geodesic(development.coordinates, development.city.coordinates.centroid).kilometers faq, _ = Faq.objects.get_or_create(development=development, category=cat4, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"{development.title} is approximately {distance:,.2f} kilometers from central {development.city.name}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"{development.title} تا مرکز {development.city.name} چه فاصله‌ای دارد؟" faq_t.answer = f"{development.title} تقریباً {distance:,.2f} کیلومتر از مرکز {development.city.name} فاصله دارد." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Как далеко {development.title} от центра {development.city.name}?" faq_t.answer = f"{development.title} расположен примерно в {distance:,.2f} километрах от центра {development.city.name}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما هي المسافة بين {development.title} و وسط {development.city.name}?" faq_t.answer = f"يبعد {development.title} حوالي {distance:,.2f} كيلومتر عن وسط {development.city.name}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"What public transportation options are available near {development.title}?" public_facilities = PublicFacility.objects.filter(type_id__in=[2,3,4,5,21,22], developments__development=development) facilities = ", ".join([f"{pf.name} {pf.type.name}" for pf in public_facilities[:5]]) if facilities: faq, _ = Faq.objects.get_or_create(development=development, category=cat4, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"Nearby transportation includes {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"گزینه‌های حمل و نقل عمومی نزدیک به {development.title} چیست؟" faq_t.answer = f"حمل و نقل نزدیک شامل {facilities} می‌شود." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Какие варианты общественного транспорта доступны рядом с {development.title}?" faq_t.answer = f"Близлежащий транспорт включает {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما هي خيارات المواصلات العامة المتاحة بالقرب من {development.title}؟" faq_t.answer = f"تشمل المواصلات القريبة {facilities}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"Are there any parks or green spaces near {development.title}?" public_facilities = PublicFacility.objects.filter(type_id__in=[17], developments__development=development) facilities = ", ".join([f"{pf.name}" for pf in public_facilities[:3]]) if facilities: faq, _ = Faq.objects.get_or_create(development=development, category=cat4, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"Yes, nearby parks include {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"آیا پارک‌ها یا فضاهای سبز نزدیک به {development.title} وجود دارد؟" faq_t.answer = f"بله، پارک‌های نزدیک شامل {facilities} می‌شوند." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Есть ли парки или зеленые зоны рядом с {development.title}?" faq_t.answer = f"Да, близлежащие парки включают {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"هل هناك حدائق أو مساحات خضراء بالقرب من {development.title}؟" faq_t.answer = f"نعم، تشمل الحدائق القريبة {facilities}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) # try: # question = f"What is the walkability score of {development.title}?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat4, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"{development.title} has a walkability score of {Walkability_Score}, making it {Walkability_Advantages}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"امتیاز پیاده‌روی {development.title} چقدر است؟" # faq_t.answer = f"{development.title} دارای امتیاز پیاده‌روی {Walkability_Score} است که آن را {Walkability_Advantages} می‌کند." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Какова оценка пешеходной доступности {development.title}?" # faq_t.answer = f"{development.title} имеет оценку пешеходной доступности {Walkability_Score}، что делает его {Walkability_Advantages}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"ما هو مؤشر سهولة المشي في {development.title}؟" # faq_t.answer = f"يمتلك {development.title} مؤشر سهولة مشي يبلغ {Walkability_Score}، مما يجعله {Walkability_Advantages}." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) try: question = f"Which schools are located near {development.title}?" public_facilities = PublicFacility.objects.filter(type_id__in=[1], developments__development=development) facilities = ", ".join([f"{pf.name}" for pf in public_facilities[:3]]) if facilities: faq, _ = Faq.objects.get_or_create(development=development, category=cat5, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"Schools nearby include {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"مدارس نزدیک به {development.title} کدامند؟" faq_t.answer = f"مدارس نزدیک شامل {facilities} می‌شوند." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Какие школы расположены рядом с {development.title}?" faq_t.answer = f"Близлежащие школы включают {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما هي المدارس الموجودة بالقرب من {development.title}؟" faq_t.answer = f"تشمل المدارس القريبة {facilities}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"Are there shopping and dining options near {development.title}?" public_facilities = PublicFacility.objects.filter(type_id__in=[12,14,19], developments__development=development) facilities = ", ".join([f"{pf.name}" for pf in public_facilities[:3]]) public_facilities = PublicFacility.objects.filter(type_id__in=[6], developments__development=development) restaurants = ", ".join([f"{pf.name}" for pf in public_facilities[:3]]) if facilities: faq, _ = Faq.objects.get_or_create(development=development, category=cat5, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"Yes, nearby options include {facilities} and restaurants like {restaurants}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"آیا گزینه‌های خرید و رستوران در نزدیکی {development.title} وجود دارد؟" faq_t.answer = f"بله، گزینه‌های خرید شامل {facilities} و رستوران‌هایی مانند {restaurants} می‌شوند." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Есть ли магазины и рестораны рядом с {development.title}?" faq_t.answer = f"Да, близлежащие варианты включают {facilities} и рестораны вроде {restaurants}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"هل توجد خيارات للتسوق والطعام بالقرب من {development.title}؟" faq_t.answer = f"نعم، تشمل الخيارات القريبة {facilities} ومطاعم مثل {restaurants}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"What healthcare facilities are close to {development.title}?" public_facilities = PublicFacility.objects.filter(type_id__in=[7], developments__development=development) facilities = ", ".join([f"{pf.name}" for pf in public_facilities[:3]]) if facilities: faq, _ = Faq.objects.get_or_create(development=development, category=cat5, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"Nearby healthcare facilities include {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"تسهیلات بهداشتی نزدیک به {development.title} کدامند؟" faq_t.answer = f"تشمل المرافق الصحية القريبة {facilities} می‌شوند." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Какие медицинские учреждения находятся близко к {development.title}?" faq_t.answer = f"Близлежащие медицинские учреждения включают {facilities}." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"ما هي المرافق الصحية القريبة من {development.title}؟" faq_t.answer = f"تشمل المرافق الصحية القريبة {facilities}." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) try: question = f"Are there any cultural or entertainment venues near {development.title}?" public_facilities = PublicFacility.objects.filter(type_id__in=[10,16], developments__development=development) facilities = ", ".join([f"{pf.name} {pf.type.name}" for pf in public_facilities[:3]]) if facilities: faq, _ = Faq.objects.get_or_create(development=development, category=cat5, question=question) faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) faq_t.question = question faq_t.answer = f"Yes, venues such as {facilities} are located nearby." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) faq_t.question = f"آیا مکان‌های فرهنگی یا تفریحی نزدیک به {development.title} وجود دارد؟" faq_t.answer = f"بله، مکان‌هایی مانند {facilities} در نزدیکی واقع شده‌اند." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) faq_t.question = f"Есть ли культурные или развлекательные заведения рядом с {development.title}?" faq_t.answer = f"Да, такие заведения как {facilities} расположены поблизости." faq_t.created_time = time.time() faq_t.save() faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) faq_t.question = f"هل هناك مواقع ثقافية أو ترفيهية بالقرب من {development.title}؟" faq_t.answer = f"نعم، توجد مواقع مثل {facilities} في المنطقة المجاورة." faq_t.created_time = time.time() faq_t.save() except Exception as e: print(e) # try: # question = f"Is {development.title} environmentally friendly?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat6, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"Yes, {development.title} includes sustainability features such as {Sustainability_Features}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"آیا {development.title} از نظر زیست‌محیطی دوستدار محیط است؟" # faq_t.answer = f"بله، {development.title} شامل ویژگی‌های پایداری مانند {Sustainability_Features} است." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Является ли {development.title} экологически чистым?" # faq_t.answer = f"Да, {development.title} включает экологические функции, такие как {Sustainability_Features}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"هل {development.title} صديق للبيئة؟" # faq_t.answer = f"نعم، يتضمن {development.title} ميزات استدامة مثل {Sustainability_Features}." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) # try: # question = f"What energy rating does {development.title} have?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat6, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"{development.title} has an energy rating of {Energy_Rating}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"{development.title} چه امتیاز انرژی دارد؟" # faq_t.answer = f"{development.title} دارای امتیاز انرژی {Energy_Rating} است." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Какой энергетический рейтинг у {development.title}?" # faq_t.answer = f"{development.title} имеет энергетический рейтинг {Energy_Rating}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"ما هو التصنيف الطاقي لـ {development.title}؟" # faq_t.answer = f"يمتلك {development.title} تصنيف طاقي يبلغ {Energy_Rating}." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) # try: # question = f"Are electric vehicle (EV) charging points available at {development.title}?" # faq, _ = Faq.objects.get_or_create(development=development, category=cat6, question=question) # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=en) # faq_t.question = question # faq_t.answer = f"Yes, EV charging points are available at {Charging_Locations}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=fa) # faq_t.question = f"آیا نقاط شارژ وسایل نقلیه برقی (EV) در {development.title} موجود است؟" # faq_t.answer = f"بله، نقاط شارژ EV در {Charging_Locations} در دسترس هستند." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ru) # faq_t.question = f"Доступны ли точки зарядки электромобилей (EV) в {development.title}?" # faq_t.answer = f"Да, точки зарядки электромобилей доступны в {Charging_Locations}." # faq_t.created_time = time.time() # faq_t.save() # faq_t, _ = FaqTranslate.objects.get_or_create(faq=faq, language=ar) # faq_t.question = f"هل تتوفر نقاط شحن للمركبات الكهربائية (EV) في {development.title}؟" # faq_t.answer = f"نعم، تتوفر نقاط شحن المركبات الكهربائية في {Charging_Locations}." # faq_t.created_time = time.time() # faq_t.save() # except Exception as e: print(e) def change_url(a_tag, country, city=None): country_slug = f"country-{country.slug}" our_url = f"/{country_slug}" if city: city_slug = f"city-{city.slug}" our_url = f"{our_url}/{city_slug}" url = a_tag.get("href") if url is None: a_tag.decompose() url = url.replace("https://1newhomes.com", "").replace("https://1newhomes.ae", "") if url.startswith("/new-homes/"): url_sections = url.split("/") if len(url_sections) <= 2: a_tag.decompose() else: add_url = "" url_mapping = { 'district-': lambda x: x, 'zone-': lambda x: x, 'station-': lambda x: x, 'options-': lambda x: x.replace("options-", "key_feature-"), 'year-finished': lambda x: "completion-ready-to-move", 'ready-to-move': lambda x: "completion-ready-to-move", 'year-': lambda x: x.replace("year-", "completion-"), 'property-studio': lambda x: "bedroom-0" } bedrooms = {f'property-{i}-bedroom': f'bedroom-{i}' for i in range(1, 10)} for i in range(2, len(url_sections)): section = url_sections[i] for prefix, transform in url_mapping.items(): if section.startswith(prefix): add_url = f"{add_url}/{transform(section)}" break for prefix, suffix in bedrooms.items(): if section.startswith(prefix): add_url = f"{add_url}/{suffix}" break if add_url != "": url = f"{our_url}{add_url}" a_tag["href"] = url else: a_tag.decompose() else: a_tag.decompose() def get_filter_page_text(): fs = FilterSeoText.objects.filter(check_for_text=False, get_text_error=False)[:30] for f in fs: try: country = f.country city = f.city if city is not None and (city.slug != "london" or city.slug != "dubai"): f.check_for_text = True f.save() continue if country.slug != "united-kingdom" and country.slug != "united-arab-emirates": f.check_for_text = True f.save() continue base_url = "" if country.slug == "united-kingdom": base_url = "https://1newhomes.com/new-homes" else: base_url = "https://1newhomes.ae/new-homes" add_url = "/" s_slug = "" first_filter_type = f.first_filter_type second_filter_type = f.second_filter_type if second_filter_type == "bedroom": if f.second_filter == "0": s_slug = "property-studio/" elif f.second_filter in ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]: s_slug = f"property-{f.second_filter}-bedroom/" else: f.check_for_text = True f.save() continue elif second_filter_type is not None and second_filter_type != "": f.check_for_text = True f.save() continue if f.first_filter: slug = slugify(f.first_filter, allow_unicode=True) if first_filter_type == "station": add_url = f"/station-{slug}/{s_slug}" elif first_filter_type == "district": add_url = f"/{slug}/{s_slug}" elif first_filter_type == "postcode": add_url = f"/{slug}/{s_slug}" elif first_filter_type == "area": if slug == "around-london": add_url = f"/{slug}/{s_slug}" else: add_url = f"/{slug}-london-area/{s_slug}" elif first_filter_type == "zone": add_url = f"/{slug}/{s_slug}" elif first_filter_type is None or first_filter_type == "": add_url = f"/{s_slug}" else: f.check_for_text = True f.save() continue url = base_url + add_url req_co = requests.get(url, timeout=10) if req_co.status_code == 200: html_content = req_co.text soup = BeautifulSoup(html_content, 'html.parser') sf_content = soup.find("div", {"class": "sf_content"}) for section in sf_content.find_all('section'): section.decompose() for section in sf_content.find_all("div", {"class": "sentence-v3-catalog"}): section.decompose() # for section in sf_content.find_all("div", {"class": "seoLinks--wrapper2"}): # section.decompose() # for section in sf_content.find_all("div", {"class": "seoLinks"}): # section.decompose() for section in sf_content.find_all('a'): change_url(section, country, city) # section.decompose() f.text = str(sf_content) print(f"{f.id}: >>>>>>>>>>>>>>>>>>>>\n{sf_content}\n<<<<<<<<<<<<<<<<<<<<<") else: f.get_text_error = True f.get_text_error_message = f"status code: {req_co.status_code}" except Exception as e: f.get_text_error = True print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} get filter page text (filter id:{f.id})error:', e) f.check_for_text = True f.save() PARAPHRASE_PROMPT = """ Paraphrase the following text, which describes a list of apartments or building projects on a real estate consulting website, into {language} language. Make sure to: 1. Use SEO-friendly language to help improve search engine rankings. 2. Avoid phrasing that closely resembles AI-generated text. 3. Maintain the professional tone of a real estate agent. 4. Retain the structure of any HTML tags present in the text. 5. Remove any empty HTML tags from the text. Original Description: `{text}` """ def paraphrase_text(text, language="english", prompt=PARAPHRASE_PROMPT, model="gpt-4o-mini"): try: client = OpenAI( api_key=openai_key, ) prompt = prompt.format(text=text, language=language) chat = client.chat.completions.create( model = model, messages = [{"role": "system", "content": ( "You are an expert real estate consultant and content optimization specialist. " "Your task is to refine and enhance user prompts to create engaging, SEO-friendly, and persuasive text for real estate listings. " "Ensure the tone matches the target audience (families, first-time buyers, investors) and the structure remains clear and sales-oriented. " "Provide actionable suggestions to help achieve the user's goals effectively." )}, {"role": "user", "content": prompt}], temperature = 0.7 ) return {"error": False, "content":chat.choices[0].message.content, "error_txt":None} except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : error : ', e) return {"error": True, "content":None, "error_txt":f"{e}"} TRANSLATE_PROMPT = """ Translate the following text into {language}, ensuring: 1. The tone and meaning of the original content are preserved. 2. SEO-friendly keywords are adapted appropriately for the target language. 3. The translated text aligns with cultural and linguistic nuances of the target audience. 4. The structure of any HTML tags present is retained. Original Text: `{text}` """ def translate_text(text,language, prompt=TRANSLATE_PROMPT,model="gpt-4o-mini"): try: client = OpenAI( api_key=openai_key, ) chat = client.chat.completions.create( model = model, messages = [{"role": "system", "content": ( "You are an expert real estate consultant and content optimization specialist. " "Your task is to refine and enhance user prompts to create engaging, SEO-friendly, and persuasive text for real estate listings. " "Ensure the tone matches the target audience (families, first-time buyers, investors) and the structure remains clear and sales-oriented. " "Provide actionable suggestions to help achieve the user's goals effectively." )}, {"role": "user", "content": prompt.format(text=text, language=language)}], temperature = 0.7 ) generated_text = chat.choices[0].message.content generated_text = generated_text.replace("```html", "").replace("```", "") return {"error": False, "content":generated_text, "error_txt":None} except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in text translate error : ', e) return {"error": True, "content":None, "error_txt":f"{e}"} def paraphrase_filter_page_text(): fs = FilterSeoText.objects.filter(check_for_text=True, get_text_error=False, text__isnull=False, is_paraphrase=False).first() if fs is None: return languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") fs_en = FilterSeoTextTranslate.objects.get_or_create(filter_seo_text=fs, language=base_language) try: text = fs.text p_text = paraphrase_text(text, language=base_language.name) if p_text["error"]: return en_text = p_text["content"] fs_en.text = en_text fs_en.save() for language in languages: fs_translate = FilterSeoTextTranslate.objects.get_or_create(filter_seo_text=fs, language=language) translate_text_result = translate_text(text, language.name) if translate_text_result["error"]: continue fs_translate.text = translate_text_result["content"] fs_translate.save() fs.is_paraphrase = True fs.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in paraphrase_filter_page_text error : ', e) def translate_filter_page_text(): languages = Language.objects.exclude(code="en") base_language = Language.objects.get(code="en") fs = FilterSeoText.objects.annotate(tr_num=Count('translates')).filter(check_for_text=True, get_text_error=False, text__isnull=False, is_paraphrase=True, tr_num__lt=languages.count()+1).first() try: fs_en = FilterSeoTextTranslate.objects.get(filter_seo_text=fs, language=base_language) except: fs.is_paraphrase = False fs.save() return try: text = fs_en.text for language in languages: fs_translate = FilterSeoTextTranslate.objects.get_or_create(filter_seo_text=fs, language=language) translate_text_result = translate_text(text, language.name) if translate_text_result["error"]: continue fs_translate.text = translate_text_result["content"] fs_translate.save() fs.is_paraphrase = True fs.save() except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in translate_filter_page_text error : ', e) def set_development_details(): developments = Development.objects.all() for development in developments: developer = development.developer currency = development.default_currency if currency is None: if development.country.slug == "united-kingdom": currency = Currency.objects.get(code="GBP") elif development.country.slug == "united-arab-emirates": currency = Currency.objects.get(code="AED") else: currency = "" development_t = development.translates.get(language__code="en") flats = development.flats.all() flats_text = "" for flat in flats: flat_t = flat.translates.get(language__code="en") flats_text += f""" {flat_t.name}: [floor_area:{flat.area} m2, price:{flat.base_price} {currency.symbol}{', [is sold out]' if flat.is_sold_out else ''}] """ development_detail, _ = DevelopmentDetail.objects.get_or_create(development=development) text = f"""reference ID: ETL-{development.id} title: {development.title} developer: {developer.name} base price: {development.base_price} {currency.symbol} country: {development.country.name} city: {development.city.name} area: {', '.join([a.name for a in development.onh_area.all()])} postcode: {', '.join([p.name for p in development.onh_postcode.all()])} district: {', '.join([d.name for d in development.onh_district.all()])} zone: {', '.join([z.name for z in development.onh_zone.all()])} station: {', '.join([s.name for s in development.onh_station.all()])} Address: {development_t.address} ( {development.address} ) [{development.coordinates.x}, {development.coordinates.y}] Floor Area: {development.area} m2 Number of Flats: {development.flat} Flats: {flats_text} Number of Floors: {development.floor} Building Type: {development.building_type.name} Nearest Station: {', '.join([s.name for s in development.onh_station.all()])} Completion Year: {development.completed_date}{' [is ready to move]' if development.is_ready_to_move else ''}{' [is sold out]' if development.is_sold_out else ''} Features: {', '.join([f.feature.name for f in development.key_features.all()])} Features Description: {development_t.features_description} Description: {development_t.description} url: https://entralon.com/en/property-{development.slug}""" development_detail.description = text development_detail.save() def create_development_detail_json(): d = DevelopmentDetail.objects.all() out = [] for development_detail in d: development = development_detail.development out.append({ "id": development.id, "price": f"{development.base_price if development.base_price > 0 else 'sold out'}", "city": development.city.name, "country": development.country.name, "reference_id": f"ETL-{development.id}", "name": development.title, "url": f"https://entralon.com/en/property-{development.slug}", "description": development_detail.description, }) file_dir = f"{settings.BASE_DIR}/media/development_detail/" if not os.path.exists(file_dir): os.makedirs(file_dir) with open(file_dir + "development_detail.json", "w") as f: json.dump(out, f)