import html
import io
import json
import re
import time
import datetime
from bs4 import BeautifulSoup
from django.conf import settings
from data_scraper.models import ONHNews, ONHNewsData, ONHNewsImage, OneNewHomesData, OneNewHomesDevelopmentsData, OneNewHomesFilterData, OneNewsHomesDeveloper
from django.core.files.images import ImageFile
import requests
from django.utils.text import slugify
from developer.models import Developer, DeveloperTranslate
from django.core.files.base import ContentFile
from development.models import Attachment, AutoScoreRank, BuildingType, Currency, Development, DevelopmentOffer, DevelopmentOfferTranslate, DevelopmentPublicFacility, DevelopmentTranslate, FaqCategory, FaqCategoryTranslate, Feature, FeatureTranslate, Flat, FlatPriceHistory, KeyFeature, ONHArea, ONHDistrict, ONHPostcode, ONHStation, ONHZone, PaymentOption, PaymentOptionTranslate, PaymentPlan, PaymentPlanTranslate, RankElement, RankElementTranslate
from geo_location.models import City, Country, Location, PublicFacility
from main.models import Language
from django.contrib.gis.geos import GEOSGeometry
from django.db.models import Q
from urllib.parse import urlparse
from openai import OpenAI
import environ
env = environ.Env()
environ.Env.read_env()
openai_key = env.str('openai_key')
openai_pr_id = env.str('openai_pr_id')
openai_org_id = env.str('openai_org_id')
def get_data_1nh():
p = 1
ap = 2
url = "https://1newhomes.com/assets/components/msearch2/action-search.php"
req_co = requests.get("https://1newhomes.com/new-homes/", timeout=30)
city = City.objects.filter(slug="london").first()
country = Country.objects.filter(slug="united-kingdom").first()
if req_co.status_code == 200:
OneNewHomesData.objects.all().delete()
OneNewHomesFilterData.objects.all().delete()
co = req_co.cookies
PHPSESSID = co['PHPSESSID']
html_content = req_co.text
OneNewHomesFilterData.objects.create(id=1, data=html.unescape(html_content), base_url="https://1newhomes.com/", city=city, country=country)
soup = BeautifulSoup(html_content, 'html.parser')
script_tag = soup.find('script', text=lambda t: t and 'mse2Config' in t)
script_content = script_tag.string.replace(",\n", ", ").replace(",\r", ", ").replace("\t","").replace(" "," ").replace(" "," ")
mse2Config_match = re.search(r'mse2Config\s*=\s*({.*})', script_content, re.DOTALL)
key = "30e35bbdbc0d5a572a05f1b709551ada3d2b5e7b"
if mse2Config_match:
mse2Config = mse2Config_match.group(1)
mse2Config_json = json.loads(mse2Config)
key = mse2Config_json.get('key')
while True:
if p > ap:
break
payload = f"page={p}&action=filter&pageId=2923&key={key}"
headers = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie': f'''linkchain=/ ### /first-time-buyers ### /new-homes/ ### / ### /first-time-buyers ### /team/ ### /new-homes/ ### /new-homes/ ### /park-modern-london ### /park-modern-london ### /park-modern-london ### / ### /park-modern-london ### / ### /goodluck-hope-london ### / ### /goodluck-hope-london ### / ### /new-homes/ ### /developers ### /bellway-london/ ### /flats/ ### /harcourt-house-london ### /new-homes/ ### /goodluck-hope-london ### /hayes-village-london ### / ### /new-homes/ ### /the-beckett-london ### /the-beckett-london/gallery ### / ### /the-silk-district-london ### /the-silk-district-london ### /the-silk-district-london ### /the-silk-district-london ### /the-silk-district-london ### /the-silk-district-london ### /hepworth-place-london ### /hepworth-place-london ### /hepworth-place-london ### /hepworth-place-london ### /countryside-properties-london/ ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /south-oxhey-central-london ### /developers ### /barratt-london/ ### /barratt-london/ ### / ### /1a-st-johns-wood-park-london ### /1a-st-johns-wood-park-london/gallery ### / ### /goodluck-hope-london ### / ### / ### / ### / ### / ### /developers ### / ### /first-time-buyers ### / ### /the-silk-district-london ### / ### /new-homes/ ### / ### / ### /new-homes/page-3 ### /new-homes/page-2 ### /new-homes/page-2 ### /new-homes/page-2 ### /new-homes/ ### /developers ### /taylor-wimpey-london/ ### /new-homes/ ### /new-homes/ ### /greenwich-millennium-village-london; _gcl_au=1.1.594264799.1719929307; _ga_JCEVRN83E0=GS1.1.1723452014.58.0.1723452016.58.0.0; _ga=GA1.1.65359444.1719929309; _ym_uid=1719929310452810339; _ym_d=1719929310; _fbp=fb.1.1722030370979.323930974364926; PHPSESSID={PHPSESSID}; cookie_agree=1; utm=; _ym_isad=1; _ym_visorc=w; PHPSESSID={PHPSESSID}'''
}
try:
response = requests.request("POST", url, headers=headers, data=payload, timeout=30)
res_json = response.json()
if res_json.get('success'):
res = res_json.get('data')
if ap < int(res['pages']):
ap = int(res['pages'])
continue
json_data = None
if p == 1:
json_data = res.get('resultsJSON')
OneNewHomesData.objects.create(id=p, base_url="https://1newhomes.com", data=html.unescape(res['results']), json_data=json_data, total=res['total'], page=res['page'], pages=res['pages'])
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {e}')
p += 1
ap = 2
p1 = 1
url = "https://1newhomes.ae/assets/components/msearch2/action-search.php"
req_co = requests.get("https://1newhomes.ae/new-homes/", timeout=30)
city = City.objects.filter(slug="dubai").first()
country = Country.objects.filter(slug="united-arab-emirates").first()
if req_co.status_code == 200:
co = req_co.cookies
PHPSESSID = co['PHPSESSID']
html_content = req_co.text
OneNewHomesFilterData.objects.create(id=2, data=html.unescape(html_content), base_url="https://1newhomes.ae/", city=city, country=country)
soup = BeautifulSoup(html_content, 'html.parser')
script_tag = soup.find('script', text=lambda t: t and 'mse2Config' in t)
script_content = script_tag.string.replace(",\n", ", ").replace(",\r", ", ").replace("\t","").replace(" "," ").replace(" "," ")
mse2Config_match = re.search(r'mse2Config\s*=\s*({.*})', script_content, re.DOTALL)
key = "30e35bbdbc0d5a572a05f1b709551ada3d2b5e7b"
if mse2Config_match:
mse2Config = mse2Config_match.group(1)
mse2Config_json = json.loads(mse2Config)
key = mse2Config_json.get('key')
while True:
if p1 > ap:
break
payload = f"page={p1}&action=filter&pageId=2923&key={key}"
headers = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie': f'''linkchain=/ ### /news/a-15-quarter-climb-in-dubai-real-estate-prices ### /developers ### /emaar-properties-dubai/ ### / ### / ### / ### /new-homes/ ### /the-address-jumeirah-resort-and-spa-dubai ### /emaar-properties-dubai/ ### /the-address-jumeirah-resort-and-spa-dubai ### / ### /new-homes/ ### /prescott-real-estate-dubai/ ### /developers ### /new-homes/; _ga_SVS4S7PFEJ=GS1.1.1726536471.6.1.1726538323.0.0.0; _ga=GA1.1.415606180.1724083972; _ym_uid=1719929312736223957; _ym_d=1724083975; PHPSESSID={PHPSESSID}; cookie_agree=1; _ym_isad=1; _ym_visorc=w'''
}
try:
response = requests.request("POST", url, headers=headers, data=payload, timeout=30)
res_json = response.json()
if res_json.get('success'):
res = res_json.get('data')
if ap < int(res['pages']):
ap = int(res['pages'])
continue
json_data = None
if p1 == 1:
json_data = res.get('resultsJSON')
OneNewHomesData.objects.create(id=p, base_url="https://1newhomes.ae", data=html.unescape(res['results']), json_data=json_data, total=res['total'], page=res['page'], pages=res['pages'])
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {e}')
p += 1
p1 += 1
def parse_json_development_data_1nh():
developments = OneNewHomesData.objects.filter(json_data_checked=False, json_data__isnull=False)
base_url = "https://1newhomes.com"
for development in developments:
data = development.json_data
error = False
for d in data:
try:
url = d.get('url')
year = d.get('year')
coord = d.get('coord')
od, _ = OneNewHomesDevelopmentsData.objects.get_or_create(slug=url)
if year:
od.year = year
od.coordinates = coord
od.save()
except:
pass
development.json_data_checked = True
development.save()
def parse_development_data_1nh():
lc = time.time() - 3600*24
developments = OneNewHomesData.objects.filter(Q(is_checked=False) | Q(last_check__lt=lc))[:10]
base_url = "https://1newhomes.com"
l = 0
city = None
country = None
for development in developments:
if development.base_url is not None: base_url = development.base_url
if "https://1newhomes.com" in base_url:
city = City.objects.filter(slug="london").first()
country = Country.objects.filter(slug="united-kingdom").first()
else:
city = City.objects.filter(slug="dubai").first()
country = Country.objects.filter(slug="united-arab-emirates").first()
soup = BeautifulSoup(development.data, 'html.parser')
development_list = soup.find_all(attrs={"class":"sentence-v3-catalog"})
l += len(development_list)
error = False
for dl in development_list:
try:
name = dl.find("h2").text
url = dl.find("a")["href"]
slug = url.split('/')[-1]
f1 = None
try:
image = base_url + dl.get('data-img')
image_file_request = requests.get(image, timeout=10)
if image_file_request.status_code == 200:
f1 = ImageFile(io.BytesIO(image_file_request.content), name='foo.png')
except Exception as e:
error = True
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {name} image save error', e)
f2 = None
builder_n = None
dev_slug = None
builder = None
builder_n = None
try:
builder_s = dl.find(attrs={"class":"builder"})['href']
dev_slug = builder_s.split('/')[-1]
if dev_slug == "": dev_slug = builder_s.split('/')[-2]
except Exception as e:
builder_n = dl.find(attrs={"class":"builder"}).text
dev_slug = slugify(builder_n, allow_unicode=True)
if dev_slug is not None and dev_slug != "":
builder = None
if dev_slug.endswith("london"):
builder = Developer.objects.filter(slug=dev_slug[:-7]).first()
else:
builder = Developer.objects.filter(slug=f"{dev_slug}-london").first()
if dev_slug.endswith("dubai"):
builder = Developer.objects.filter(slug=dev_slug[:-6]).first()
elif builder is None:
builder = Developer.objects.filter(slug=f"{dev_slug}-dubai").first()
if dev_slug.endswith("developers"):
builder = Developer.objects.filter(slug=dev_slug[:-11]).first()
elif builder is None:
builder = Developer.objects.filter(slug=f"{dev_slug}-developers").first()
if dev_slug.endswith("group"):
builder = Developer.objects.filter(slug=dev_slug[:-6]).first()
elif builder is None:
builder = Developer.objects.filter(slug=f"{dev_slug}-group").first()
if builder is None:
builder = Developer.objects.filter(slug=dev_slug).first()
if builder is None:
builder = Developer.objects.create(slug=dev_slug)
if builder.name is None or builder.name == "":
if builder_n is not None: builder.name = builder_n.title()
else: builder.name = dev_slug.replace('-', " ").title()
builder.city = city
builder.country = country
builder.save()
language, _ = Language.objects.get_or_create(name="english", code="en")
dev_tr, _ = DeveloperTranslate.objects.get_or_create(developer=builder, language=language)
dev_tr.name=name
dev_tr.slug=slug
dev_tr.save()
if builder is None:
print(f'on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {name} builder not Found builder name={builder_n} and slug={dev_slug}')
try:
if base_url.endswith(".com"):
address = dl.find("span", attrs={"class":"building-addr"}).text
elif base_url.endswith(".ae"):
address = dl.find("span", attrs={"class":"district"}).text
except: address = None
try:
od, _ = OneNewHomesDevelopmentsData.objects.get_or_create(slug=slug)
od.name = name
od.url = url
od.base_url = base_url
if f1 is not None: od.image = f1
od.builder = builder
od.builder_slug = dev_slug
od.builder_name = builder_n
od.address = address
od.data_downloaded = False
od.save()
try:
req = requests.get(url, timeout=30)
if req.status_code == 200:
od.url_data = html.unescape(req.text)
od.data_downloaded = True
od.save()
except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {name} error in download page: ', e)
try:
req = requests.get(f"{url}/gallery", timeout=30)
if req.status_code == 200:
od.gallery_data = html.unescape(req.text)
od.gallery_downloaded = True
od.gallery_last_download = time.time()
od.save()
except Exception as e: print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {name} error in download gallery: ', e)
except Exception as e:
error = True
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {name} error in save to data base: ', e)
except Exception as e:
error = True
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in development list [{development.id}]: ', e)
if not error:
development.is_checked = True
development.last_check = time.time()
development.save()
print(f'in {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {l} developments checked')
def download_development_data_1nh():
developments = OneNewHomesDevelopmentsData.objects.filter(data_downloaded=False)[:10]
for development in developments:
try:
url = development.url
req = requests.get(url, timeout=30)
if req.status_code == 200:
development.url_data = html.unescape(req.text)
development.data_downloaded = True
development.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : development {development.name}:', e)
def download_development_gallery_1nh():
lt = time.time() - 10 * 24 * 60 * 60
developments = OneNewHomesDevelopmentsData.objects.filter(Q(gallery_downloaded=False) | Q(gallery_last_download__lt=lt))[:10]
for development in developments:
try:
url = f"{development.url}/gallery"
req = requests.get(url, timeout=30)
if req.status_code == 200:
development.gallery_data = html.unescape(req.text)
development.gallery_downloaded = True
development.gallery_last_download = time.time()
development.gallery_check=False
development.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : download gallery page of development {development.name}:', e)
def create_public_facility(objs, type, development):
for obj in objs:
try:
obj_name = obj.get('name')
if "\\\\u" in obj_name:
try: obj_name = obj_name.replace("\\\\","\\").encode("ascii").decode("unicode-escape")
except:
try:
obj_name = obj_name.replace("\\\\","\\").encode("utf8").decode("unicode-escape")
except: pass
obj_slug = slugify(obj_name[:150], allow_unicode=True)
try: obj_distance = float(obj.get('distance'))
except: obj_distance = None
try: obj_duration = obj.get('duration')
except: obj_duration = None
obj_location = obj.get('geometry').get('location')
pnt = GEOSGeometry(f'POINT({obj_location.get("lng")} {obj_location.get("lat")})')
new_location, _ = Location.objects.get_or_create(coordinates=pnt)
new_location.name= obj_name[:150]
new_location.slug = obj_slug
new_location.save()
pf, _ = PublicFacility.objects.get_or_create(coordinates=pnt, location=new_location, type_id=type)
pf.name = obj_name[:150]
pf.slug = obj_slug
pf.save()
dpf, _ = DevelopmentPublicFacility.objects.get_or_create(development=development, public_facility=pf)
dpf.distance = obj_distance
dpf.duration = obj_duration
dpf.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : create public facility {obj_name}:', e)
def create_development():
language, _ = Language.objects.get_or_create(name="english", code="en")
if OneNewHomesDevelopmentsData.objects.filter(in_check=True).count() > 90:
OneNewHomesDevelopmentsData.objects.all().update(in_check=False)
lc = time.time() - 3600*24
developments = OneNewHomesDevelopmentsData.objects.filter(Q(is_checked=False) | Q(last_check__lt=lc), in_check=False, name__isnull=False)[:40]
for d in developments:
d.in_check=True
d.is_checked=False
d.save()
for development in developments:
error = False
err = None
url = development.url
name = development.name
slug = slugify(name, allow_unicode=True)
# print(name, development.id)
try:
if url.startswith('https://1newhomes.com/'):
default_currency = Currency.objects.get(code='GBP')
city = City.objects.get(slug='london')
country = Country.objects.get(slug='united-kingdom')
elif url.startswith('https://1newhomes.ae/'):
default_currency = Currency.objects.get(code='AED')
city = City.objects.get(slug='dubai')
country = Country.objects.get(slug='united-arab-emirates')
else:
default_currency = None
city = None
country = None
new_file = None
try:
new_file = ContentFile(development.image.read())
new_file.name = development.image.name
except: new_file = None
try:
new_file1 = ContentFile(development.image_mobile.read())
new_file1.name = development.image_mobile.name
except: new_file1 = None
try:
try:
html_code = development.url_data
soup = BeautifulSoup(html_code, 'html.parser')
except:
req = requests.get(url, timeout=30)
html_code = html.unescape(req.text)
soup = BeautifulSoup(html_code, 'html.parser')
except Exception as e:
print(f'error on read html: {e}')
development.gallery_in_check = False
development.save()
continue
description = None
try:
des = soup.find(attrs={"class":"tab-description"}).find(attrs={"class":"collapsed-text"})
description = "".join([f"{item}" for item in des.contents])
except: pass
try:
description = description.replace("1newhomes", "Entralon").replace("\\n","").replace("\n","").replace("\\r","").replace("\r","").replace("\\t","").replace("\\'","'").replace("\\/","/").strip()
except: pass
f_description = None
try:
des = soup.find(attrs={"class":"tab-features"}).find(attrs={"class":"collapsed-text"})
f_description = "".join([f"{item}" for item in des.contents])
except: pass
try:
f_description = f_description.replace("1newhomes", "Entralon").replace("\\n","").replace("\n","").replace("\\r","").replace("\r","").replace("\\t","").replace("\\'","'").replace("\\/","/").strip()
except: pass
neighborhood = None
try:
neighborhood = soup.find(attrs={"class":"neighborhood-content"})
neighborhood.find('a').decompose()
except: pass
try:
neighborhood = "".join([f"{item}" for item in neighborhood.contents])
neighborhood = neighborhood.replace("1newhomes", "Entralon").replace("\\n","").replace("\n","").replace("\\r","").replace("\r","").replace("\\t","").replace("\\'","'").replace("\\/","/").strip()
except: pass
area_range = None
min_area = 0
max_area = 0
try:
completed_date = development.year
if completed_date is None: completed_date = soup.find(attrs={"class":"completion"}).find(attrs={"class":"value"}).text
elif completed_date == "finished": completed_date = "Ready to move"
completed_date = completed_date.replace("\\n","").replace("\n","").replace("\\r","").replace("\r","").replace("\\t","").replace("\\'","'").replace("\\/","/").replace(" \\xe2\\x80\\x93 ", " - ").strip()
except: completed_date = None
if completed_date is None or completed_date == "": is_ready_to_move = True
else: is_ready_to_move = completed_date.lower() in ["ready to move","readytomove"]
now = datetime.datetime.now()
completed_at = None
if not is_ready_to_move:
cd = completed_date.lower()
ar = []
if " – " in cd:
ar = cd.split(" – ")
elif " xe2x80x93 " in cd:
ar = cd.split(" xe2x80x93 ")
elif " - " in cd:
ar = cd.split(" - ")
else:
ar.append(cd)
for i in ar:
month = 12
day = 30
if "q/" in i:
ad = i.split("q/",1)
q = ad[0]
year = ad[1]
if q == "1":
month = 3
day = 31
elif q == "2":
month = 6
day = 30
elif q == "3":
month = 9
day = 30
elif q == "4":
month = 12
day = 31
else:
day = 31
month = 12
year = i
ca = datetime.date(int(year), month, day)
if completed_at is None: completed_at = ca
elif ca > completed_at: completed_at = ca
if completed_at is not None and completed_at.year < now.year: is_ready_to_move = True
try: p_ts = soup.find(attrs={"class":"property-type"}).find(attrs={"class":"value"}).find_all(attrs={"class":"station"})
except: p_ts = []
station_types = {
"u":["subway-stations"],
"r":["rail-stations"],
"o":["train-stations"],
"d":["dlr-stations"],
"ur":["subway-stations","rail-stations"],
"rd":["rail-stations","dlr-stations"],
"uo":["subway-stations","train-stations"],
"urd":["subway-stations","rail-stations","dlr-stations"],
"ro":["rail-stations","train-stations"],
"rod":["rail-stations","train-stations","dlr-stations"],
"ud":["subway-stations","dlr-stations"],
"uro":["subway-stations","rail-stations","train-stations"],
"urd":["subway-stations","rail-stations","dlr-stations"],
}
try: station_div = soup.select('.prop.station')[0].find_all(attrs={"class":"value"})
except: station_div = []
station_top = []
for i in station_div:
i_c = i.get('class')
name = html.unescape(i.text)
name = name.replace("\\n","").replace("\n","").replace("\\r","").replace("\r","").replace("\\t","").replace("\\'","'").replace("\\/","/").strip()
if "station-u" in i_c:
station_top.append({"type":station_types["u"], "name":name})
try:
area_range = soup.find(attrs={"class":"square"}).find(attrs={"class":"value"}).text
except: pass
if area_range:
if area_range.lower().endswith("sqm"):
p = 0.09290304
else: p = 1
area_range = area_range.replace("sqft", "").replace("sqm", "").split(" – ")
try: min_area = float(area_range[0]) * p
except: pass
try: max_area = float(area_range[-1]) * p
except:pass
bedrooms_range = []
try:
bedrooms_input = soup.find(attrs={"class":"f-count"}).find_all('input')
bedrooms_label = soup.find(attrs={"class":"f-count"}).find_all('label')
if bedrooms_input:
for i in range(len(bedrooms_input)):
try: data_num = bedrooms_label[i].text if bedrooms_label[i].text.lower() != "s" else "0"
except: data_num = "0"
try:
data_sqm = float(bedrooms_input[i].get('data-sqm'))
except: data_sqm = 0
try:
data_all = float(bedrooms_input[i].get('data-all'))
except:
data_all = 0
try: data_area = data_all / data_sqm
except: data_area = min_area if data_num == "0" else max_area
bedrooms_range.append({
"num": data_num,
"sqm": data_sqm,
"all": data_all,
"area": data_area,
"soldout": data_sqm == 0
})
except: pass
is_sold_out = False
try:
is_sold_out = soup.find(attrs={"class":"price-block--value"}).find(attrs={"class":"button"}).text.lower() in ["soldout", "sold out"]
except:
try: is_sold_out = soup.find(attrs={"class":"price-block--soldout"}).find(attrs={"class":"value"}).text.lower() in ["soldout", "sold out"]
except: pass
# flat_price_div = soup.find('section', attrs={"id":"price"})
property_offers = soup.find_all(attrs={"class":"property-offer"})
payment_plans = soup.find_all(attrs={"class":"payment-plan--item"})
payment_methods = soup.find_all(attrs={"class":"payment-method"})
key_features = soup.find_all(attrs={"class":"tth_table--column"})
auto_scores = soup.find_all(attrs={"class":"expert--circle-wrapper"})
faqs_groups = soup.find_all(attrs={"class":"faq-wrapper"})
address = ""
location = None
places = None
prices = None
prices_history = []
script_tag = soup.find('script', text=lambda t: t and 'page_config' in t)
script_price = soup.find('script', text=lambda t: t and 'priceSelectorData' in t)
if script_price:
script_price = script_price.string.replace("\\","\\\\")
priceSelectorData_match = re.search(r'const priceSelectorData\s*=\s*(\[.*\])', script_price, re.DOTALL)
if priceSelectorData_match:
prices = priceSelectorData_match.group(1)
prices = json.loads(prices)
for price in prices:
try:
if price.get('rooms')[0].lower() == "s": bdr_num_data = "0"
elif price.get('rooms')[0].lower() == "v": bdr_num_data = "10000" # villa
elif price.get('rooms')[0].lower() == "t": bdr_num_data = "10001" # townhouse
elif price.get('rooms')[0].lower() == "p": bdr_num_data = "10002" # penthouse
elif price.get('rooms')[0].lower() == "d": bdr_num_data = "10003" # duplex
else: bdr_num_data = price.get('rooms')[0]
except: bdr_num_data = "0"
try: bdr_sqm_data = float(price.get('priceM2', 0))
except: bdr_sqm_data = 0
try: bdr_all_data = float(price.get('priceWhole', 0))
except: bdr_all_data = 0
try: bdr_area_data = float(price.get('area', 0))
except: bdr_area_data = 0
bedrooms_range.append({
"num": bdr_num_data,
"sqm": bdr_sqm_data,
"all": bdr_all_data,
"area": bdr_area_data,
"soldout": price.get('soldout') != ""
})
coordinates = None
if script_tag:
# script_content = script_tag.string.replace("\\","\\\\")
script_content = script_tag.string.replace(",\n", ", ").replace(",\r", ", ").replace("\t","").replace(" "," ").replace(" "," ").replace('\\"',"'").replace("\\","\\\\")
address_match = re.search(r'"address"\s*:\s*"(.*?)",', script_content, re.DOTALL)
if address_match:
address = address_match.group(1)
infrastructurePlaces_match = re.search(r'const infrastructurePlaces\s*=\s*({.*})', script_content, re.DOTALL)
if infrastructurePlaces_match:
places = infrastructurePlaces_match.group(1)
try: places = json.loads(places)
except Exception as e:
places = None
historyPricesData_match = re.search(r'const historyPricesData\s*=\s*(\[.*\])\\\\n\\\\n ', script_content, re.DOTALL)
if historyPricesData_match:
prices_history = historyPricesData_match.group(1)
prices_history = re.sub(r"new Date\((\d+)\)",r"\1", prices_history )
prices_history = prices_history.replace("'", '"').replace('name:', '"name":').replace('data:', '"data":').replace('x:', '"x":').replace('y:', '"y":').replace("\\\\","")
prices_history = json.loads(prices_history)
lng = 0.0
lat = 0.0
try:
coordinates_long_match = re.search(r'"coordinates_long"\s*:\s*(.*?),', script_content, re.DOTALL)
if coordinates_long_match:
lng = coordinates_long_match.group(1)
coordinates_lat_match = re.search(r'"coordinates_lat"\s*:\s*(.*?),', script_content, re.DOTALL)
if coordinates_lat_match:
lat = coordinates_lat_match.group(1)
coordinates= GEOSGeometry(f'POINT({lng} {lat})')
location, create = Location.objects.get_or_create(coordinates=coordinates)
if create:
location.name= name
location.save()
except: pass
try:
try: new_development = Development.objects.get(slug=slug)
except:
new_development = Development.objects.create(slug=slug)
if new_development.title is None: new_development.title = name
if new_development.developer is None: new_development.developer = development.builder
if new_development.image is None and new_file1 is not None: new_development.image = new_file
if new_development.image_mobile is None and new_file1 is not None: new_development.image_mobile = new_file1
if new_development.min_area is None or new_development.min_area == 0: new_development.min_area = min_area
if new_development.max_area is None or new_development.max_area == 0: new_development.max_area = max_area
# if new_development.location is None:
new_development.location = location
new_development.area = (new_development.max_area + new_development.min_area) / 2
new_development.is_sold_out = is_sold_out
new_development.places = places
new_development.is_ready_to_move = is_ready_to_move
if coordinates: new_development.coordinates = coordinates
if default_currency: new_development.default_currency = default_currency
if city: new_development.city = city
if country: new_development.country = country
if completed_date: new_development.completed_date = completed_date
if completed_at: new_development.completed_at = completed_at
new_development.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} create new development:', e)
development.gallery_in_check = False
development.save()
continue
try: development_translate = DevelopmentTranslate.objects.get(language=language, development=new_development)
except: development_translate = DevelopmentTranslate.objects.create(language=language, development=new_development)
if development_translate.title is None: development_translate.title=name
if development_translate.slug is None: development_translate.slug=slug
if development_translate.description is None:
if description: development_translate.description=description
if development_translate.features_description is None:
if f_description: development_translate.features_description=f_description
if neighborhood: development_translate.neighborhood=neighborhood
if development_translate.address is None: development_translate.address=address
development_translate.save()
for pt in p_ts:
try:
pt_name = pt.text.lower().replace(r"\\n\s+", "").title()
pt_slug = slugify(pt_name, allow_unicode=True)
try: property_type = BuildingType.objects.get(slug=pt_slug)
except: property_type = BuildingType.objects.create(name=pt_name, slug=pt_slug)
new_development.building_type.add(property_type)
except Exception as e: pass
for bedroom in bedrooms_range:
try:
try: bedroom_num = int(bedroom.get('num'))
except: bedroom_num = 0
new_flat, create_flat = Flat.objects.get_or_create(development=new_development, bedrooms_num=bedroom_num)
if create_flat: new_flat.area = bedroom.get('area')
if new_flat.area == 0: new_flat.area = bedroom.get('area')
new_flat.price_per_meter = bedroom.get('sqm')
new_flat.base_price = bedroom.get('all')
new_flat.is_sold_out = bedroom.get('soldout') == True
new_flat.save()
fph = FlatPriceHistory.objects.filter(flat=new_flat).order_by('time').last()
if (fph is None or fph.price != bedroom.get('sqm')) and bedroom.get('sqm', 0) > 0:
fph = FlatPriceHistory.objects.create(flat=new_flat, price=bedroom.get('sqm'), time=time.time()*1000)
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} create new flat by data {bedroom}:', e)
for ph in prices_history:
if ph['name'].lower() in ["studio", ""]:
bn = "0"
elif ph['name'].lower() in ["villa"]:
bn = "10000"
elif ph['name'].lower() in ["townhouse"]:
bn = "10001"
elif ph['name'].lower() in ["penthouse"]:
bn = "10002"
elif ph['name'].lower() in ["duplex"]:
bn = "10003"
else:
bn = ph['name'][0]
try: flat,_ = Flat.objects.get_or_create(development=new_development, bedrooms_num=bn)
except: continue
ph_data = ph['data']
for d in ph_data:
FlatPriceHistory.objects.get_or_create(flat=flat, price=d['y'], time=d['x'])
for offer in property_offers:
try:
po_name = offer.find('p').text.replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
do, _ = DevelopmentOffer.objects.get_or_create(development=new_development, text=po_name)
try: DevelopmentOfferTranslate.objects.get(language=language, development_offer=do)
except: DevelopmentOfferTranslate.objects.create(language=language, development_offer=do, text=po_name)
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} create new offer {offer.text}:', e)
for payment_plan in payment_plans:
try:
plan_percent = payment_plan.find(attrs={"class":"payment-plan--value"}).text[:-1]
try: plan_percent = float(plan_percent.replace(",", "."))
except: plan_percent = 0
plan_title = payment_plan.find(attrs={"class":"payment-plan--title"}).text.replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
plan_slug = slugify(plan_title, allow_unicode=True)
try: do = PaymentPlan.objects.get(development=new_development, slug=plan_slug)
except: do = PaymentPlan.objects.create(development=new_development, title=plan_title)
do.percent=plan_percent
do.save()
try: PaymentPlanTranslate.objects.get(language=language, payment_plan=do)
except: PaymentPlanTranslate.objects.create(language=language, payment_plan=do, title=plan_title)
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} create new payment plan {payment_plan.text}:', e)
for payment_method in payment_methods:
try:
po_name = payment_method.text.replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
po_slug = slugify(po_name)
do, _ = PaymentOption.objects.get_or_create(name=po_name, slug=po_slug)
if 'active' in payment_method.get('class', []):
new_development.payment_option.add(do)
try: PaymentOptionTranslate.objects.get(language=language, payment_option=do)
except: PaymentOptionTranslate.objects.create(language=language, payment_option=do, name=po_name, slug=po_slug)
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} create new payment method {payment_method.text}:', e)
for key_feature in key_features:
try:
kf_icon = key_feature.find('svg').prettify()
kf_title = key_feature.find(attrs={"class":"title"}).text.replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
kf_slug = slugify(kf_title, allow_unicode=True)
try: kf_value = key_feature.find(attrs={"class":"tth-text"}).text
except: kf_value = None
try: kf_value = kf_value.replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
except: pass
if "floors" in kf_title.lower():
try:
floor = kf_value.replace(" ", "")
new_development.floor = floor
floors = floor.split("-")
new_development.min_floor = floors[0]
new_development.max_floor = floors[-1]
new_development.save()
except Exception as e: print("add floor: ", e)
if "flats" in kf_title.lower():
try:
p_flat = kf_value.replace(" ", "")
new_development.flat = p_flat
p_flats = p_flat.split("-")
new_development.min_flat = p_flats[0]
new_development.max_flat = p_flats[-1]
new_development.save()
except Exception as e: print("add flat: ", e)
if "property type" in kf_title.lower():
try:
pt_name = kf_value.lower().replace(r"\\n\s+", "").title()
pt_slug = slugify(pt_name, allow_unicode=True)
try: property_type = BuildingType.objects.get(slug=pt_slug)
except: property_type = BuildingType.objects.create(name=pt_name, slug=pt_slug)
new_development.building_type.add(property_type)
except Exception as e: print("add property type: ",e)
try: kf_num = float(kf_value)
except: kf_num = None
try: new_f = Feature.objects.get(slug=kf_slug)
except: new_f = Feature.objects.create(slug=kf_slug, name=kf_title)
if new_f.icon_text is None: new_f.icon_text = kf_icon
new_f.save()
try: FeatureTranslate.objects.get(language=language, feature=new_f)
except: FeatureTranslate.objects.create(language=language, feature=new_f, name=kf_title, slug=kf_slug)
new_kf, _ = KeyFeature.objects.get_or_create(development=new_development, feature=new_f)
new_kf.value=kf_value
new_kf.num=kf_num
new_kf.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} create new key feature {key_feature.kf_title}:', e)
fg_order = 0
for faqs_group in faqs_groups:
try:
faqs_group_title = faqs_group.find('h3').text.replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
try:
category, _ = FaqCategory.objects.get_or_create(title=faqs_group_title)
except: category = FaqCategory.objects.create(title=faqs_group_title)
category.order = fg_order
category.save()
fg_order += 1
try: FaqCategoryTranslate.objects.get(language=language, category=category)
except: FaqCategoryTranslate.objects.create(language=language, category=category, title=faqs_group_title)
# questions = faqs_group.find_all('div', attrs={"class":"faq-question"})
# for question in questions:
# faq_title = question.find(attrs={"class":"faq-title"}).text.replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
# faq_answer = question.find(attrs={"class":"faq-answer"}).text.replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
# try:
# faq = Faq.objects.get(development=new_development, question=faq_title, category=category)
# except:
# faq = Faq.objects.create(development=new_development, question=faq_title, answer=faq_answer, category=category)
# try: FaqTranslate.objects.get(language=language, faq=faq)
# except: FaqTranslate.objects.create(language=language, faq=faq, question=faq_title, answer=faq_answer, created_time=time.time())
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} create new faqs group:', e)
for auto_score in auto_scores:
try:
as_icon = auto_score.find('svg').prettify().replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
as_title = auto_score.find(attrs={"class":"expert--circle-title"}).text.replace("\\n","").replace(":","").replace("\\r","").replace("\\t","").strip()
as_slug = slugify(as_title, allow_unicode=True)
as_value = None
try: as_value = auto_score.find(attrs={"class":"expert--circle"}).get('data-value')
except Exception as e: print(e)
try: new_re = RankElement.objects.get(slug=as_slug)
except: new_re = RankElement.objects.create(slug=as_slug, name=as_title)
if new_re.icon_text is None: new_re.icon_text = as_icon
new_re.save()
try: RankElementTranslate.objects.get(language=language, rank_element=new_re)
except: RankElementTranslate.objects.create(language=language, rank_element=new_re, name=as_title, slug=as_slug)
new_asr, _ = AutoScoreRank.objects.get_or_create(development=new_development, element=new_re)
if as_value: new_asr.rank = as_value
new_asr.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} create new auto score rank {key_feature.text}:', e)
for st in station_top:
try:
st_slug = slugify(st['name'][:150], allow_unicode=True)
pf, _ = PublicFacility.objects.get_or_create(slug=st_slug, type_id=st['type'])
pf.name = st['name']
pf.rank = 1
pf.is_active = True
pf.save()
dpf, _ = DevelopmentPublicFacility.objects.get_or_create(development=new_development, public_facility=pf)
dpf.rank = 1
dpf.save()
except: pass
try:
parks = places.get('park', {}).get('results')
create_public_facility(parks, 17, new_development)
except:pass
try:
cafes = places.get('cafe', {}).get('results')
create_public_facility(cafes, 13, new_development)
except:pass
try:
primary_schools = places.get('primary_school', {}).get('results')
create_public_facility(primary_schools, 1, new_development)
except:pass
try:
bus_stations = places.get('bus_station', {}).get('results')
create_public_facility(bus_stations, 4, new_development)
except:pass
try:
gyms = places.get('gym', {}).get('results')
create_public_facility(gyms, 18, new_development)
except:pass
try:
hospitals = places.get('hospital', {}).get('results')
create_public_facility(hospitals, 7, new_development)
except:pass
try:
supermarkets = places.get('supermarket', {}).get('results')
create_public_facility(supermarkets, 19, new_development)
except:pass
try:
train_stations = places.get('train_station', {}).get('results')
create_public_facility(train_stations, 3, new_development)
except:pass
try:
subway_stations = places.get('subway_station', {}).get('results')
create_public_facility(subway_stations, 2, new_development)
except:pass
# airports = places.get('airports', {}).get('rows')
# create_public_facility(airports, 5)
flats = list(new_development.flats.all().values("bathrooms_num", "bedrooms_num", "is_sold_out", "area", "price_per_meter", "base_price"))
try: developer = {"name":new_development.developer.name, "slug": f"developer-{new_development.developer.slug}", "image": f"{new_development.developer.image}", "id":new_development.developer.id}
except: developer = None
try: currency = {"name":new_development.default_currency.name, "code": new_development.default_currency.code, "symbol": new_development.default_currency.symbol}
except: currency = None
geo_json = {
"type": "Feature",
"id": new_development.id,
"properties": {
"id": new_development.id,
"title": new_development.title,
"slug": f"property-{new_development.slug}",
"base_price": new_development.base_price,
"address": new_development.address,
"area": new_development.area,
"min_area": new_development.min_area,
"max_area": new_development.max_area,
"floor": new_development.floor,
"flat": new_development.flat,
"is_featured": new_development.is_featured,
"completed_date": new_development.completed_date,
"is_ready_to_move": new_development.is_ready_to_move,
"is_sold_out": new_development.is_sold_out,
"image": f"{new_development.image}",
"flats": flats,
"developer":developer,
"currency": currency,
},
"geometry": {
"type": "Point",
"coordinates": [
new_development.coordinates.x,
new_development.coordinates.y
]
}
}
new_development.geo_json = geo_json
new_development.last_create = time.time()
new_development.save()
except Exception as e:
error = True
err = e
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} create:', e)
if not error:
development.is_checked = True
development.last_check = time.time()
development.in_check = False
development.save()
else:
development.is_checked = True
development.last_check = time.time()
development.have_error = True
development.error = f"{err}"
development.in_check = False
development.save()
def add_attachment_to_development():
base_url = "https://1newhomes.com"
if OneNewHomesDevelopmentsData.objects.filter(gallery_in_check=True).count() > 50:
OneNewHomesDevelopmentsData.objects.all().update(gallery_in_check=False)
lt = time.time() * 1 * 24 * 60 * 60
developments = OneNewHomesDevelopmentsData.objects.filter(Q(gallery_check=False) | Q(gallery_last_check__lt=lt), gallery_downloaded=True, gallery_in_check=False)[:20]
for d in developments:
d.gallery_check=False
d.gallery_in_check=True
d.save()
for development in developments:
if development.base_url is not None: base_url = development.base_url
if development.gallery_data is None or development.gallery_data == "":
development.gallery_downloaded = False
development.save()
continue
try:
data = development.gallery_data
soup = BeautifulSoup(data, 'html.parser')
divs = soup.find_all(attrs={"class":"media-gallery--wrapper"})
slug = slugify(development.name, allow_unicode=True)
dev = Development.objects.get(slug=slug)
for div in divs:
category = 0
try:
div_id = div.get('id')
if div_id:
if div_id.lower() == "interior": category = 2
elif div_id.lower() == "site-plan": category = 5
elif div_id.lower() == "exterior": category = 1
figures = div.find_all('figure')
for figure in figures:
img = figure.find('img')
src = img['src']
title = img.get('title', development.name)
alt = img.get('alt', development.name)
file_name = "-".join(src.split("/")[-1].split(".")[:-1])
try:
figcaption = figure.find('figcaption').text
if figcaption:
if "primary photo" in figcaption.lower() : category = 0
elif "exterior" in figcaption.lower(): category = 1
elif "interior" in figcaption.lower(): category = 2
elif "amenities" in figcaption.lower(): category = 3
elif "floor plan" in figcaption.lower(): category = 4
elif "site plan" in figcaption.lower(): category = 5
elif "flat plan" in figcaption.lower(): category = 6
elif "layouts" in figcaption.lower(): category = 6
except: figcaption = img.get('title')
f1 = None
try:
image = base_url + src
image_file_request = requests.get(image, timeout=10)
image_name = f'{slug}-{src.split("/")[-1]}'
if image_file_request.status_code == 200:
f1 = ImageFile(io.BytesIO(image_file_request.content), name=image_name)
try: Attachment.objects.get(file_name=file_name, development=dev, category=category)
except:
try: Attachment.objects.create(file_name=file_name, development=dev, title=title, alt=alt, image=f1, category=category)
except Exception as e: pass
development.gallery_check = True
development.gallery_last_check = time.time()
development.gallery_in_check = False
development.save()
dev.save()
except Exception as e:
development.gallery_in_check = False
development.save()
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : {development.name} image save error', e)
except Exception as e:
development.gallery_in_check = False
development.save()
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} development {development.name} attachment:', e)
except Exception as e:
development.gallery_in_check = False
development.save()
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} attachment for development {development.name} not created:', e)
def parse_filter_data_1nh():
lc = time.time() - 3600*24
filter = OneNewHomesFilterData.objects.filter(Q(is_checked=False) | Q(last_check__lt=lc)).first()
if filter:
try:
html_code = filter.data
city = filter.city
country = filter.country
html = BeautifulSoup(html_code, 'html.parser')
script_tag = html.find('script', attrs={"id":"geoselector_modal_template"}).text
soup = BeautifulSoup(script_tag, 'html.parser')
stations = soup.find_all("span", attrs={"data-tv":"subway"})
for station in stations:
name = station.text
slug = slugify(name, allow_unicode=True)
code = station.get('data-value')
try:
s_type = re.sub(r"\d", "", code)
try:
st = ONHStation.objects.get(slug=slug, code=code, city=city, country=country)
except:
st = ONHStation.objects.create(name=name, code=code, slug=slug, city=city, country=country)
st.type = f"station-{s_type}"
st.icon = f"icons/station-{s_type}.svg"
st.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} station {name} not created:', e)
districts = soup.find_all("span", attrs={"data-tv":"district"})
for district in districts:
name = district.text
slug = slugify(name, allow_unicode=True)
code = district.get('data-value')
try:
st = ONHDistrict.objects.get_or_create(name=name, code=code, slug=slug, city=city, country=country)
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} district {name} not created:', e)
postcodes = soup.find_all("span", attrs={"data-tv":"postcode"})
for postcode in postcodes:
name = postcode.text
slug = slugify(name, allow_unicode=True)
code = postcode.get('data-value')
try:
st = ONHPostcode.objects.get_or_create(name=name, code=code, slug=slug, city=city, country=country)
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} postcode {name} not created:', e)
try: areas = html.find(attrs={"class":"area"}).find("ul").find_all("li")
except: areas = []
for area in areas:
name = area.find('label').text
slug = slugify(name, allow_unicode=True)
code = area.find('input').get('value')
try:
st = ONHArea.objects.get_or_create(name=name, code=code, slug=slug, city=city, country=country)
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} area {name} not created:', e)
try: zones = html.find(attrs={"class":"zone"}).find("ul").find_all("li")
except: zones = []
for zone in zones:
name = zone.find('label').text
slug = slugify(name, allow_unicode=True)
code = zone.find('input').get('value')
try:
st = ONHZone.objects.get_or_create(name=name, code=code, slug=slug, city=city, country=country)
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} zone {name} not created:', e)
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} filter {filter.id} not created:', e)
filter.is_checked = True
filter.last_check = time.time()
filter.save()
def get_data_1nh_developer():
base_url = "https://1newhomes.com"
url = f"{base_url}/developers"
req = requests.get(url, timeout=30)
html_page = html.unescape(req.text)
soup = BeautifulSoup(html_page, 'html.parser')
developers_list = soup.find_all(attrs={"class":"developers-list--item"})
city = City.objects.get(slug="london")
for dl in developers_list:
try:
name = dl.div.div.string
link = f"{base_url}{dl['href']}"
slug = dl['href'][1:-1]
image = f"{base_url}{dl.img['src']}"
f = None
try:
image_file_request = requests.get(image, timeout=30)
if image_file_request.status_code == 200:
f = ImageFile(io.BytesIO(image_file_request.content), name='foo.png')
except Exception as e:
print("image save error", e)
od, _ = OneNewsHomesDeveloper.objects.get_or_create(slug=slug)
od.name = name
od.url = link
od.city = city
od.image = f
od.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} developer:', e)
base_url = "https://1newhomes.ae"
url = f"{base_url}/developers"
req = requests.get(url, timeout=30)
html_page = html.unescape(req.text)
soup = BeautifulSoup(html_page, 'html.parser')
developers_list = soup.find_all(attrs={"class":"developers-list--item"})
city = City.objects.get(slug="dubai")
for dl in developers_list:
try:
name = dl.div.div.string
link = f"{base_url}{dl['href']}"
slug = dl['href'][1:-1]
image = f"{base_url}{dl.img['src']}"
f = None
try:
image_file_request = requests.get(image, timeout=30)
if image_file_request.status_code == 200:
f = ImageFile(io.BytesIO(image_file_request.content), name='foo.png')
except Exception as e:
print("image save error", e)
od, _ = OneNewsHomesDeveloper.objects.get_or_create(slug=slug)
od.name = name
od.url = link
od.city = city
od.image = f
od.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} developer:', e)
def create_developer():
developers = OneNewsHomesDeveloper.objects.filter(is_checked=False)[:50]
language, _ = Language.objects.get_or_create(name="english", code="en")
for developer in developers:
try:
name = developer.name
link = developer.url
city = developer.city
country = city.country
slug = slugify(name, allow_unicode=True)
new_file = ContentFile(developer.image.read())
new_file.name = developer.image.name
req = requests.get(link, timeout=15)
if req.status_code == 200:
html_page = html.unescape(req.text)
soup = BeautifulSoup(html_page, 'html.parser')
try:
website = soup.find(attrs={"class":"btn ga"})['href']
try:
parsed_url = urlparse(website)
website = f"{parsed_url.scheme}://{parsed_url.netloc}"
except:
website = None
except: website = None
des = soup.find(attrs={"class":"sticky-block--content"})
try:
h1 = des.find(attrs={"class":"title-content"})
h1.replaceWith('')
except: pass
try:
img = des.find(attrs={"class":"developer-logo"})
img.replaceWith('')
except: pass
try:
off = des.find(attrs={"class":"developer-offers"})
off.replaceWith('')
except: pass
description = f"{des}"
try:
description = description.replace("1newhomes", "Entralon").replace("\\n","").replace("\n","").replace("\\r","").replace("\r","").replace("\\t","").replace("\\'","'").replace("\\/","/").strip()
except: pass
dev = None
if slug.endswith("london"):
dev = Developer.objects.filter(slug=slug[:-7]).first()
else:
dev = Developer.objects.filter(slug=f"{slug}-london").first()
if slug.endswith("dubai"):
dev = Developer.objects.filter(slug=slug[:-6]).first()
elif dev is None:
dev = Developer.objects.filter(slug=f"{slug}-dubai").first()
if slug.endswith("developers"):
dev = Developer.objects.filter(slug=slug[:-11]).first()
elif dev is None:
dev = Developer.objects.filter(slug=f"{slug}-developers").first()
if slug.endswith("group"):
dev = Developer.objects.filter(slug=slug[:-6]).first()
elif dev is None:
dev = Developer.objects.filter(slug=f"{slug}-group").first()
if dev is None:
dev = Developer.objects.filter(slug=slug).first()
if dev is None:
dev = Developer.objects.create(slug=slug)
dev.name=name
dev.image = new_file
dev.web_address = website
dev.city = city
dev.country = country
dev.save()
dev_tr, _ = DeveloperTranslate.objects.get_or_create(developer=dev, language=language)
dev_tr.name=name
dev_tr.slug=slug
if not dev_tr.description: dev_tr.description = description
dev_tr.save()
developer.is_checked = True
developer.last_check = time.time()
developer.save()
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} developer {developer.name}:', e)
def delete_temp_files():
import os
import time
base_dir = settings.BASE_DIR
folder_path = f"{base_dir}/media/temp_files/developer"
age_threshold = 2
now = time.time()
for root, dirs, files in os.walk(folder_path, topdown=False):
for name in files:
file_path = os.path.join(root, name)
file_age = now - os.path.getmtime(file_path)
if file_age > age_threshold * 86400:
try:
os.remove(file_path)
except OSError as e:
print(f"Error deleting {file_path}: {e}")
for name in dirs:
dir_path = os.path.join(root, name)
if not os.listdir(dir_path):
try:
os.rmdir(dir_path)
except OSError as e:
print(f"Error deleting folder {dir_path}: {e}")
folder_path = f"{base_dir}/media/temp_files/development"
for root, dirs, files in os.walk(folder_path, topdown=False):
for name in files:
file_path = os.path.join(root, name)
file_age = now - os.path.getmtime(file_path)
if file_age > age_threshold * 86400:
try:
os.remove(file_path)
except OSError as e:
print(f"Error deleting {file_path}: {e}")
for name in dirs:
dir_path = os.path.join(root, name)
if not os.listdir(dir_path):
try:
os.rmdir(dir_path)
except OSError as e:
print(f"Error deleting folder {dir_path}: {e}")
def get_news_data_1nh():
p = 65
total_1 = 64
total_2 = 8
url_1 = "https://1newhomes.com"
url_2 = "https://1newhomes.ae"
city_1 = City.objects.filter(slug="london").first()
city_2 = City.objects.filter(slug="dubai").first()
ONHNewsData.objects.all().delete()
while True:
p -= 1
if p == 0:
break
try:
r = requests.get(f"{url_1}/news/?page={p}")
if r.status_code == 200:
ONHNewsData.objects.create(id=p, city=city_1, base_url=url_1, data=r.text, page=p, total=total_1)
else:
total_1 -= 1
if p > 8: continue
r = requests.get(f"{url_2}/news/?page={p}")
if r.status_code == 200:
ONHNewsData.objects.create(id=p+64, city=city_2, base_url=url_2, data=r.text, page=p, total=total_2)
else:
total_2 -= 1
except:
pass
def parse_onh_news():
onh_news_data = ONHNewsData.objects.filter(is_checked=False)[:1]
for onh_news in onh_news_data:
error = False
base_url = onh_news.base_url
city = onh_news.city
soup = BeautifulSoup(onh_news.data, "html.parser")
news_list = soup.find_all("div", class_="news-block--wrapper")
for news in news_list:
try:
title = news.find("div", class_="title").text
slug = slugify(title, allow_unicode=True)
date = news.find("div", class_="date").text
url = news.find("a", class_="news-block")["href"]
summary = news.find("div", class_="announce").text
group = news.find("span", class_="news-block--rubric").text
c_url = f"{base_url}{url}"
r = requests.get(c_url, timeout=10)
if r.status_code == 200:
soup = BeautifulSoup(r.text, "html.parser")
content = soup.find("div", class_="sticky-block--content")
if content:
image = content.find("img", class_="basepage--cover-fullwidth")["src"]
f1 = None
if image:
image_url = f"{base_url}{image}"
image_file_request = requests.get(image_url, timeout=10)
image_name = f'{slug}-{image.split("/")[-1]}'
if image_file_request.status_code == 200:
f1 = ImageFile(io.BytesIO(image_file_request.content), name=image_name)
text = content.find("div", class_="news--main-content")
for a in text.findAll('a', href=True):
a.extract()
onh_n_news = None
try:
onh_n_news = ONHNews.objects.get(url=c_url)
except:
onh_n_news = ONHNews.objects.create(
title=title,
slug=slug,
city=city,
date=date,
url=c_url,
summary=summary,
group=group,
image=f1,
description="".join([f"{item}" for item in text.contents]).replace("1newhomes", "Entralon")
)
add_image = False
for img in text.findAll('img'):
text_image = img["src"]
if text_image:
image_file_request = requests.get(text_image, timeout=10)
image_name = f'{slug}-{text_image.split("/")[-1]}'
if image_file_request.status_code == 200:
f2 = ImageFile(io.BytesIO(image_file_request.content), name=image_name)
try:
onh_n_news_image = ONHNewsImage.objects.get(news=onh_n_news, url=text_image)
except:
onh_n_news_image = ONHNewsImage.objects.create(news=onh_n_news, url=text_image, image=f2)
add_image = True
img['src']=f"{onh_n_news_image.image}"
if add_image:
onh_n_news.description="".join([f"{item}" if item != "\n" else "" for item in text.contents])
onh_n_news.is_rephrase = False
onh_n_news.save()
except Exception as e:
error = True
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in get news detail [{onh_news.id}] error : ', e)
if not error:
onh_news.is_checked = True
onh_news.last_check = time.time()
onh_news.save()
PARAPHRASE_PROMPT = """paraphrase the following text, which news text on a real estate consulting website, into {language} language. Use HTML tags and headings h3-h6 if applicable. Maintain the tone of a professional real estate agent.
Original Text: `{text}`
"""
def paraphrase_text(text, title, language="english", prompt=PARAPHRASE_PROMPT, model="gpt-4o-mini"):
try:
client = OpenAI(
api_key=openai_key,
)
prompt = prompt.format(text=text, language=language)
chat = client.chat.completions.create(
model = model,
messages = [{"role": "system", "content": "You are an experienced real estate consultant with deep market knowledge."},
{"role": "user", "content": prompt}],
temperature = 0.7
)
return {"error": False, "content":chat.choices[0].message.content, "error_txt":None}
except Exception as e:
print(f'error on {datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")} : in news text paraphrase [{title}] error : ', e)
return {"error": True, "content":None, "error_txt":f"{e}"}
def onh_news_paraphrase():
news = ONHNews.objects.filter(is_paraphrase=False, paraphrase_error=False).first()
text = news.description
title = news.title
if text:
response = paraphrase_text(text, title)
if not response["error"]:
news.description = response["content"].replace("```html","").replace("```","")
news.is_paraphrase = True
news.save()
else:
news.paraphrase_error = True
news.save()