329 lines
10 KiB
Python
329 lines
10 KiB
Python
|
import os
|
||
|
|
import io
|
||
|
|
import csv
|
||
|
|
import traceback
|
||
|
|
from datetime import datetime
|
||
|
|
from flask import current_app
|
||
|
|
from webinterface import db
|
||
|
|
from webinterface.models.gtfs_static import Agencia, Servicio, Paradero, Parada, Ruta, Linea
|
||
|
|
|
||
|
|
def parse_time(strinput):
|
||
|
|
try:
|
||
|
|
h, m, s = strinput.split(":")
|
||
|
|
h = int(h)
|
||
|
|
d = 0
|
||
|
|
while h > 23:
|
||
|
|
d += 1
|
||
|
|
h -= 24
|
||
|
|
|
||
|
|
return datetime.strptime("{}:{}:{}".format(h,m,s), '%H:%M:%S').time(), d
|
||
|
|
except:
|
||
|
|
current_app.logger.debug('Traceback {}'.format(traceback.format_exc()))
|
||
|
|
current_app.logger.warning("Hora incorrecta => {}".format(strinput))
|
||
|
|
return datetime.strptime("01:01:01", '%H:%M:%S').time(), 0
|
||
|
|
|
||
|
|
|
||
|
|
def parse_paraderoid(strinput):
|
||
|
|
primerodenero2022 = 44562
|
||
|
|
|
||
|
|
if strinput.find("-") > 0:
|
||
|
|
current_app.logger.error("Error en el dato de la parada: {}".format(strinput))
|
||
|
|
try:
|
||
|
|
whatdate= datetime.strptime('{}-{}'.format(strinput, 2022), '%d-%b-%Y').date()
|
||
|
|
idparadero = primerodenero2022 + whatdate.timetuple().tm_yday
|
||
|
|
current_app.logger.warning("Calculado el Identificador en base al año 2022 => {}".format(idparadero))
|
||
|
|
return idparadero
|
||
|
|
except:
|
||
|
|
return 0
|
||
|
|
else:
|
||
|
|
return int(strinput)
|
||
|
|
|
||
|
|
|
||
|
|
def completa_dataset():
|
||
|
|
iparaderos = 0
|
||
|
|
for paradero in Paradero.query.all():
|
||
|
|
iparaderos += 1
|
||
|
|
cparadero = 0
|
||
|
|
lparadas = 0
|
||
|
|
sparadas = 0
|
||
|
|
dparadas = 0
|
||
|
|
lineas = []
|
||
|
|
lparada = []
|
||
|
|
|
||
|
|
for parada in Parada.query.filter(Parada.paraderoid==paradero.id).all():
|
||
|
|
cparadero += 1
|
||
|
|
if cparadero % 1000 == 0:
|
||
|
|
current_app.logger.debug("Paradero {}, {} detenciones. {} lineas".format(iparaderos, cparadero, len(lineas)))
|
||
|
|
|
||
|
|
elemento = (parada.servicio.lineaid, parada.servicio.direccionid)
|
||
|
|
|
||
|
|
if parada.servicio.dotw == 'S':
|
||
|
|
sparadas += 1
|
||
|
|
elif parada.servicio.dotw == 'D':
|
||
|
|
dparadas += 1
|
||
|
|
else:
|
||
|
|
lparadas += 1
|
||
|
|
|
||
|
|
if elemento in lineas:
|
||
|
|
continue
|
||
|
|
|
||
|
|
lineas.append(elemento)
|
||
|
|
if parada.servicio.direccionid:
|
||
|
|
lparada.append("{} ({} ida)".format(parada.servicio.linea.codigo, parada.secuencia))
|
||
|
|
else:
|
||
|
|
lparada.append("{} ({} regreso)".format(parada.servicio.linea.codigo, parada.secuencia))
|
||
|
|
|
||
|
|
lparada.sort()
|
||
|
|
paradero.recorridos = " - ".join(lparada)
|
||
|
|
paradero.paradas = 5*lparadas+sparadas+dparadas
|
||
|
|
paradero.resumen = "{} paradas a la semana, {} lineas".format(paradero.paradas, len(lineas))
|
||
|
|
paradero.paradasl = lparadas
|
||
|
|
paradero.paradass = sparadas
|
||
|
|
paradero.paradasd = dparadas
|
||
|
|
paradero.lineas = len(lineas)
|
||
|
|
current_app.logger.info("Paradero {}, {} detenciones a la semana en {} lineas".format(iparaderos, paradero.paradas, len(lineas)))
|
||
|
|
if iparaderos % 10 == 0:
|
||
|
|
db.session.commit()
|
||
|
|
|
||
|
|
db.session.commit()
|
||
|
|
|
||
|
|
def agrega_paraderos():
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
from zipfile import ZipFile
|
||
|
|
|
||
|
|
# print(os.getcwd())
|
||
|
|
|
||
|
|
current_app.logger.info("Archivo de paraderos, kmz")
|
||
|
|
with ZipFile('webinterface/models/datos/PARADEROS_FORMALES.kmz', 'r') as kmz:
|
||
|
|
kml = kmz.open(kmz.filelist[0].filename, 'r').read()
|
||
|
|
|
||
|
|
soup = BeautifulSoup(kml, 'xml')
|
||
|
|
existen = 0
|
||
|
|
nuevo = 0
|
||
|
|
|
||
|
|
|
||
|
|
for paradero in soup.find_all("kml:Placemark"):
|
||
|
|
if (existen+nuevo) % 1000 == 0:
|
||
|
|
current_app.logger.debug("Leidos {} paraderos existentes y {} nuevos".format(existen, nuevo))
|
||
|
|
|
||
|
|
|
||
|
|
lat = int(paradero.find("kml:SimpleData", {"name": "LATITUDE"}).text)/1000000
|
||
|
|
lng = int(paradero.find("kml:SimpleData", {"name": "LONGITUDE"}).text)/1000000
|
||
|
|
|
||
|
|
codigo = paradero.find("kml:SimpleData", {"name": "CODIGO_INT"}).text
|
||
|
|
comuna = paradero.find("kml:SimpleData", {"name": "COMUNA"}).text
|
||
|
|
recorridos = paradero.find("kml:SimpleData", {"name": "RECORRIDOS"}).text
|
||
|
|
tamano = paradero.find("kml:SimpleData", {"name": "SUPERFICIE"}).text
|
||
|
|
capacidad = paradero.find("kml:SimpleData", {"name": "CAPACIDAD"}).text
|
||
|
|
materialpiso = paradero.find("kml:SimpleData", {"name": "RADIER_MAT"}).text
|
||
|
|
materialpared = paradero.find("kml:SimpleData", {"name": "PARED_POS2"}).text
|
||
|
|
materialtecho = paradero.find("kml:SimpleData", {"name": "TECHUMBRE2"}).text
|
||
|
|
|
||
|
|
|
||
|
|
match = None
|
||
|
|
|
||
|
|
for radio in [0.00000001, 0.00000002, 0.00000003, 0.00000004, 0.00000005, 0.000000075, 0.0000001, 0.0000002]:
|
||
|
|
match = Paradero.query.filter(((Paradero.latitud-lat)*(Paradero.latitud-lat)+(Paradero.longitud-lng)*(Paradero.longitud-lng)) < radio).first()
|
||
|
|
|
||
|
|
if match is not None:
|
||
|
|
break
|
||
|
|
|
||
|
|
if match is None:
|
||
|
|
nuevo += 1
|
||
|
|
nombre = "{} cerca {}".format(paradero.find("kml:SimpleData", {"name": "NOMBRE_DE_"}).text, paradero.find("kml:SimpleData", {"name": "CALLE_CERC"}).text)
|
||
|
|
match = Paradero(id=codigo, codigo=codigo, nombre=nombre, latitud=lat, longitud=lng)
|
||
|
|
db.session.add(match)
|
||
|
|
db.session.commit()
|
||
|
|
else:
|
||
|
|
existen += 1
|
||
|
|
|
||
|
|
match.codigo = codigo
|
||
|
|
match.comuna = comuna
|
||
|
|
match.recorridos2 = recorridos
|
||
|
|
match.tamano = tamano
|
||
|
|
match.capacidad = capacidad
|
||
|
|
match.techo = materialtecho
|
||
|
|
match.pared = materialpared
|
||
|
|
match.piso = materialpiso
|
||
|
|
db.session.commit()
|
||
|
|
|
||
|
|
current_app.logger.info("Cargados {} paraderos existentes y {} nuevos".format(existen, nuevo))
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def load_dataset():
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
from zipfile import ZipFile
|
||
|
|
|
||
|
|
with ZipFile('GC.zip', 'r') as dataset:
|
||
|
|
current_app.logger.debug("Abriendo archivo GC.zip")
|
||
|
|
|
||
|
|
#Carga las agencias:
|
||
|
|
with io.TextIOWrapper(dataset.open('agency.txt'), encoding='utf8') as agencias_csv:
|
||
|
|
current_app.logger.debug("Leyendo agency.txt")
|
||
|
|
agencias = csv.reader(agencias_csv, delimiter=',')
|
||
|
|
elementos = 0
|
||
|
|
|
||
|
|
for item in agencias:
|
||
|
|
if elementos > 0:
|
||
|
|
agencia = Agencia.query.filter(Agencia.id==item[0]).one_or_none()
|
||
|
|
|
||
|
|
if agencia is None:
|
||
|
|
agencia = Agencia(id=item[0], nombre=item[1], url=item[2], fono=item[5])
|
||
|
|
db.session.add(agencia)
|
||
|
|
else:
|
||
|
|
agencia.nombre = item[1]
|
||
|
|
agencia.url = item[2]
|
||
|
|
agencia.fono = item[3]
|
||
|
|
|
||
|
|
elementos += 1
|
||
|
|
|
||
|
|
db.session.commit()
|
||
|
|
current_app.logger.info("Se cargaron {} agencias".format(elementos))
|
||
|
|
|
||
|
|
|
||
|
|
#Carga las agencias
|
||
|
|
with io.TextIOWrapper(dataset.open('stops.txt'), encoding='utf8') as paraderos_csv:
|
||
|
|
current_app.logger.debug("Leyendo stops.txt")
|
||
|
|
paraderos = csv.reader(paraderos_csv, delimiter=',')
|
||
|
|
elementos = 0
|
||
|
|
|
||
|
|
for item in paraderos:
|
||
|
|
if elementos > 0:
|
||
|
|
|
||
|
|
idparadero = item[0]
|
||
|
|
|
||
|
|
paradero = Paradero.query.filter(Paradero.id==idparadero).one_or_none()
|
||
|
|
if len(item[1]) > 0:
|
||
|
|
codigo = item[1]
|
||
|
|
else:
|
||
|
|
codigo = None
|
||
|
|
|
||
|
|
if paradero is None:
|
||
|
|
|
||
|
|
paradero = Paradero(id=idparadero, codigo=codigo, nombre=item[2], latitud=item[4], longitud=item[5])
|
||
|
|
db.session.add(paradero)
|
||
|
|
else:
|
||
|
|
paradero.codigo = codigo
|
||
|
|
paradero.nombre = item[2]
|
||
|
|
paradero.latitud = item[4]
|
||
|
|
paradero.longitud = item[5]
|
||
|
|
|
||
|
|
elementos += 1
|
||
|
|
if elementos % 1000 == 0:
|
||
|
|
current_app.logger.info("Se han cargado {} elementos".format(elementos))
|
||
|
|
|
||
|
|
db.session.commit()
|
||
|
|
current_app.logger.info("Se cargaron {} paradas".format(elementos))
|
||
|
|
|
||
|
|
|
||
|
|
with io.TextIOWrapper(dataset.open('routes.txt'), encoding='utf8') as lineas_csv:
|
||
|
|
current_app.logger.debug("Leyendo routes.txt")
|
||
|
|
lineas = csv.reader(lineas_csv, delimiter=',')
|
||
|
|
elementos = 0
|
||
|
|
|
||
|
|
for item in lineas:
|
||
|
|
if elementos > 0:
|
||
|
|
variante = Linea.query.filter(Linea.id==item[0]).one_or_none()
|
||
|
|
|
||
|
|
if variante is None:
|
||
|
|
variante = Linea(id=item[0], agenciaid=item[1], codigo=item[2], nombre=item[3], color=item[7])
|
||
|
|
db.session.add(variante)
|
||
|
|
else:
|
||
|
|
variante.agenciaid = item[1]
|
||
|
|
variante.codigo = item[2]
|
||
|
|
variante.nombre = item[3]
|
||
|
|
variante.color = item[7]
|
||
|
|
|
||
|
|
elementos += 1
|
||
|
|
if elementos % 1000 == 0:
|
||
|
|
current_app.logger.info("Se han cargado {} elementos".format(elementos))
|
||
|
|
|
||
|
|
db.session.commit()
|
||
|
|
current_app.logger.info("Se cargaron {} Linea".format(elementos))
|
||
|
|
|
||
|
|
|
||
|
|
with io.TextIOWrapper(dataset.open('trips.txt'), encoding='utf8') as rutas_csv:
|
||
|
|
current_app.logger.debug("Leyendo Servicios.txt")
|
||
|
|
rutas = csv.reader(rutas_csv, delimiter=',')
|
||
|
|
elementos = 0
|
||
|
|
|
||
|
|
for item in rutas:
|
||
|
|
if elementos > 0:
|
||
|
|
|
||
|
|
itinerario = Servicio.query.filter(Servicio.id==item[2], Servicio.lineaid==item[0]).one_or_none()
|
||
|
|
|
||
|
|
if itinerario is None:
|
||
|
|
itinerario = Servicio(id=item[2], lineaid=item[0], direccionid=item[5], destino=item[3], dotw=item[1], rutaid=item[7])
|
||
|
|
db.session.add(itinerario)
|
||
|
|
else:
|
||
|
|
itinerario.destino = item[3]
|
||
|
|
itinerario.direccionid = item[5]
|
||
|
|
itinerario.rutaid = item[7]
|
||
|
|
itinerario.dotw = item[1]
|
||
|
|
|
||
|
|
elementos += 1
|
||
|
|
if elementos % 1000 == 0:
|
||
|
|
db.session.commit()
|
||
|
|
current_app.logger.info("Se han cargado {} elementos".format(elementos))
|
||
|
|
|
||
|
|
db.session.commit()
|
||
|
|
current_app.logger.info("Se cargaron {} servicios".format(elementos))
|
||
|
|
|
||
|
|
with io.TextIOWrapper(dataset.open('shapes.txt'), encoding='utf8') as shapes_csv:
|
||
|
|
current_app.logger.debug("Leyendo rutas.txt")
|
||
|
|
shapes = csv.reader(shapes_csv, delimiter=',')
|
||
|
|
elementos = 0
|
||
|
|
|
||
|
|
for item in shapes:
|
||
|
|
if elementos > 0:
|
||
|
|
shape = Ruta.query.filter(Ruta.rutaid==item[0], Ruta.secuencia==item[3]).one_or_none()
|
||
|
|
|
||
|
|
if shape is None:
|
||
|
|
shape = Ruta(rutaid=item[0], secuencia=item[3], latitud=item[1], longitud=item[2])
|
||
|
|
db.session.add(shape)
|
||
|
|
else:
|
||
|
|
shape.latitud = item[1]
|
||
|
|
shape.longitud = item[2]
|
||
|
|
|
||
|
|
elementos += 1
|
||
|
|
if elementos % 1000 == 0:
|
||
|
|
db.session.commit()
|
||
|
|
current_app.logger.info("Se han cargado {} elementos".format(elementos))
|
||
|
|
|
||
|
|
|
||
|
|
db.session.commit()
|
||
|
|
current_app.logger.info("Se cargaron {} puntos".format(elementos))
|
||
|
|
|
||
|
|
with io.TextIOWrapper(dataset.open('stop_times.txt'), encoding='utf8') as paradas_csv:
|
||
|
|
current_app.logger.debug("Leyendo paradas.txt")
|
||
|
|
paradas = csv.reader(paradas_csv, delimiter=',')
|
||
|
|
elementos = 0
|
||
|
|
|
||
|
|
for item in paradas:
|
||
|
|
if elementos > 0:
|
||
|
|
idparadero = item[3]
|
||
|
|
|
||
|
|
parada = Parada.query.filter(Parada.servicioid==item[0], Parada.paraderoid==idparadero, Parada.secuencia==item[4]).one_or_none()
|
||
|
|
llegada, _ = parse_time(item[1])
|
||
|
|
salida, otrodia = parse_time(item[2])
|
||
|
|
|
||
|
|
if parada is None:
|
||
|
|
parada = Parada(servicioid=item[0], paraderoid=idparadero, secuencia=item[4], tipo=item[8], llegada=llegada, salida=salida, otrodia=otrodia)
|
||
|
|
db.session.add(parada)
|
||
|
|
else:
|
||
|
|
parada.llegada = llegada
|
||
|
|
parada.salida = salida
|
||
|
|
parada.otrodia = otrodia
|
||
|
|
parada.tipo = item[8]
|
||
|
|
|
||
|
|
elementos += 1
|
||
|
|
if elementos % 1000 == 0:
|
||
|
|
db.session.commit()
|
||
|
|
current_app.logger.info("Se han cargado {} elementos".format(elementos))
|
||
|
|
|
||
|
|
db.session.commit()
|
||
|
|
current_app.logger.info("Se cargaron {} paradas".format(elementos))
|