first commit

.gitignore

@@ -0,0 +1,2 @@
+*pyc
+static/**

Dockerfiles/Dockerfile.api

@@ -0,0 +1,27 @@
+FROM python:3-slim
+
+RUN apt-get update \
+    && apt-get -y install libpq-dev gcc \
+    && pip install psycopg2
+
+# set a directory for the app
+COPY apiweb/requeriments.txt /srv
+
+# install dependencies
+RUN pip3 install --no-cache-dir -r /srv/requeriments.txt
+
+COPY web.py /srv
+COPY static /srv/static
+COPY apiweb /srv/apiweb
+COPY utils /srv/utils
+
+# define the port number the container should expose
+RUN groupadd app && useradd -g app app
+RUN chown -R app:app /srv
+USER app
+WORKDIR /srv
+EXPOSE 8000
+
+# run the command
+ENTRYPOINT ["gunicorn"]
+CMD ["--timeout", "600", "-b", "0.0.0.0:8000", "web:iapp"]

Dockerfiles/Dockerfile.fetcher

@@ -0,0 +1,23 @@
+FROM python:3-slim
+
+RUN apt-get update \
+    && apt-get -y install libpq-dev gcc \
+    && pip install psycopg2
+
+# install dependencies
+COPY fetcher/requeriments.txt /srv
+RUN pip3 install --no-cache-dir -r /srv/requeriments.txt
+
+# copy all the files to the container
+COPY static /srv/static
+COPY fetcher /srv/fetcher
+COPY fetch.py /srv
+
+WORKDIR /srv
+RUN useradd -m fetch
+RUN chown -R fetch:fetch /srv
+USER fetch
+
+# run the command
+ENTRYPOINT ["python"]
+CMD ["fetch.py"]

Dockerfiles/Dockerfile.updater

@@ -0,0 +1,24 @@
+FROM python:3-slim
+
+RUN apt-get update \
+    && apt-get -y install libpq-dev gcc \
+    && pip install psycopg2
+
+# install dependencies
+COPY updater/requeriments.txt /srv
+RUN pip3 install --no-cache-dir -r /srv/requeriments.txt
+
+# copy all the files to the container
+COPY updater /srv/updater
+COPY static /srv/static
+COPY utils /srv/utils
+COPY sync.py /srv
+
+WORKDIR /srv
+RUN useradd -m updater
+RUN chown -R updater:updater /srv
+USER updater
+
+# run the command
+ENTRYPOINT ["python"]
+CMD ["sync.py"]

apiweb/config.py

@@ -0,0 +1,10 @@
+import os
+
+
+class Config:
+    SECRET_KEY = os.environ.get('SECRET_KEY')
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
+    SQLALCHEMY_ECHO = False
+    DEBUG = os.environ.get('DEBUG')
+
+    SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')

apiweb/content/errors/handlers.py

@@ -0,0 +1,17 @@
+from flask import Blueprint, render_template
+
+errors = Blueprint('errors', __name__)
+
+@errors.app_errorhandler(404)
+def error_404(error):
+    return render_template('errors/404.html'), 404
+
+
+@errors.app_errorhandler(403)
+def error_403(error):
+    return render_template('errors/403.html'), 403
+
+
+@errors.app_errorhandler(500)
+def error_500(error):
+    return render_template('errors/500.html'), 500

apiweb/content/main.py

@@ -0,0 +1,33 @@
+from flask import render_template, send_file, Blueprint
+from apiweb.model.feed import Registros, Vehiculo, Viaje, Entidades, Posicion
+from apiweb.main import db
+from time import strftime
+import os
+
+
+main = Blueprint('main', __name__)
+
+@main.route("/gtfs-rt/api")
+def home():
+    records = Registros.query.order_by(Registros.id.desc()).limit(100).all()
+    return render_template('summary.html', title='Historial', records=records)
+
+
+@main.route("/gtfs-rt/api/concepcion")
+@main.route("/gtfs-rt/api/concepcion/<int:registro_id>")
+def concepcion(registro_id=None):
+    if registro_id is None:
+        record = Registros.query.order_by(Registros.id.desc()).first()
+        outfile = "concepcion_gtfs-rt.proto"
+    else:
+        record = Registros.query.filter(Registros.id==registro_id).one_or_none()
+
+    if record is None:
+        abort(404)
+
+    if id is None:
+        outfile = "concepcion_gtfs-rt.latest.proto"
+    else:
+        outfile = "concepcion_gtfs-rt.{}.proto".format(record.timestamp.strftime("%H%md_%H%M_%S"))
+
+    return send_file(os.path.abspath(record.filename), download_name=outfile)

apiweb/main.py

@@ -0,0 +1,26 @@
+# coding: utf-8
+from flask import Flask
+from flask.logging import default_handler
+from flask_sqlalchemy import SQLAlchemy
+import logging
+import sys
+
+from .config import Config
+
+db = SQLAlchemy()
+
+def create_app(config_class=Config):
+    app = Flask(__name__)
+
+    app.config.from_object(config_class)
+
+    db.init_app(app)
+
+    from apiweb.content.main import main
+    from apiweb.content.errors.handlers import errors
+
+    app.register_blueprint(main)
+    app.register_blueprint(errors)
+
+
+    return app

apiweb/model/feed.py

@@ -0,0 +1,60 @@
+# coding: utf-8
+from sqlalchemy.dialects import postgresql, sqlite
+from sqlalchemy.sql import func
+from apiweb.main import db
+
+class Registros(db.Model):
+    __tablename__ = 'registros'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = db.Column(db.Integer(), primary_key=True, autoincrement=True)
+
+    filename  = db.Column(db.String())
+    status    = db.Column(db.Integer(), default=0)
+    timestamp = db.Column(db.DateTime(timezone=True), default=func.now())
+
+class Vehiculo(db.Model):
+    __tablename__ = 'vehicle'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = db.Column(db.Integer(), primary_key=True, autoincrement=True)
+
+    patente   = db.Column(db.String(8))
+    timestamp = db.Column(db.BigInteger())
+
+class Viaje(db.Model):
+    __tablename__ = 'trip'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = db.Column(db.Integer(), primary_key=True, autoincrement=True)
+
+    trip_id      = db.Column(db.String(50))
+    route_id     = db.Column(db.Integer())
+    direction_id = db.Column(db.Integer())
+
+    start_time   = db.Column(db.String(8))
+    start_date   = db.Column(db.String(8))
+
+class Entidades(db.Model):
+    __tablename__ = 'entity'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id         = db.Column(db.Integer(), primary_key=True, autoincrement=True)
+    entity     = db.Column(db.String(50))
+
+    vehicleid  = db.Column(db.Integer(), db.ForeignKey('gtfsrt.vehicle.id'), nullable=False)
+    tripid     = db.Column(db.Integer(), db.ForeignKey('gtfsrt.trip.id'), nullable=True)
+    timestamp  = db.Column(db.BigInteger())
+
+class Posicion(db.Model):
+    __tablename__ = 'datapoint'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = db.Column(db.Integer(), primary_key=True, autoincrement=True)
+
+    latitude  = db.Column(db.Float)
+    longitude = db.Column(db.Float)
+
+    bearing   = db.Column(db.Integer())
+    odometer  = db.Column(db.Float)
+    speed     = db.Column(db.Float)
+
+    vehicleid = db.Column(db.Integer(), db.ForeignKey('gtfsrt.vehicle.id'), nullable=False)
+    tripid    = db.Column(db.Integer(), db.ForeignKey('gtfsrt.trip.id'), nullable=True)
+
+    timestamp = db.Column(db.BigInteger())

apiweb/requeriments.txt

@@ -0,0 +1,9 @@
+Flask
+Flask-SQLAlchemy
+SQLAlchemy
+gunicorn
+psycopg2
+protobuf
+ua-parser
+user-agents
+netaddr

apiweb/templates/_paging.html

@@ -0,0 +1,23 @@
+{% macro render_pagination(pagination, endpoint) %}
+  <div class="container-fluid"> Página: 
+  {% if pagination.has_prev %}
+   <a href="{{ url_for(endpoint, page=pagination.page-1)}}">Anterior</a>
+  {% endif %}
+  
+  {% for page in pagination.iter_pages() %}
+    {% if page %}
+      {% if page != pagination.page %}
+        <a class="btn btn-secondary btn-sm btn-outline-info" href="{{ url_for(endpoint, page=page) }}">{{ page }}</a>
+      {% else %}
+        <strong>{{ page }}</strong>
+      {% endif %}
+    {% else %}
+      <span class=ellipsis>…</span>
+    {% endif %}	
+  {% endfor %}
+  
+  {% if pagination.has_next %}
+   <a href="{{ url_for(endpoint, page=pagination.page+1)}}">Siguiente</a>
+  {% endif %}
+  </div>
+{% endmacro %}

apiweb/templates/about.html

@@ -0,0 +1,12 @@
+{% extends "layout.html" %}
+{% block content %}
+        <article class="media content-section">
+
+          <div class="media-body">
+            <legend class="border-bottom mb-4">Acerca de la plataforma</legend>
+            
+            <p class="article-content">Esta es una plataforma de desarrollo. </p>
+            <p class="article-content">Esta plataforma no está pensada para ser utilizada directamente sino para usarla como base de desarrollos del sistema iLab.</p>
+          </div>
+        </article>
+{% endblock content %}

apiweb/templates/dashboard.html

@@ -0,0 +1,8 @@
+{% extends "layout.html" %}
+{% block content %}
+<div class="d-sm-flex align-items-center justify-content-between mb-4 col-12 px-4 row">
+<h1 class="h3 mb-1 text-gray-800 col-12">Sitio en Desarrollo</h1>
+<p class="mb-4 col-12">Bienvenido,</p>
+<p class="mb-4 col-12">Actualmente nos encontramos desarrollando esta página, asi que es muy posible que no funcione correctamente. Cualquier problema que usted detecte, puede mencionarselo a israel.figueroa@ilab.cl para que lo corrija cuando tenga tiempo.</p>
+</div>
+{% endblock content %}

apiweb/templates/errors/403.html

@@ -0,0 +1,9 @@
+{% extends "layout.html" %}
+{% block content %}
+    <div class="col-12 text-center">
+      <div class="error mx-auto" data-text="403">403</div>
+      <p class="lead text-gray-800 mb-5">Permisos insuficientes</p>
+      <p class="text-gray-500 mb-0">Usted no está autorizado a realizar esa operación.</p>
+      <a href="{{ url_for('main.home') }}">&larr; Volver al Dashboard</a>
+    </div>
+{% endblock content %}

apiweb/templates/errors/404.html

@@ -0,0 +1,9 @@
+{% extends "layout.html" %}
+{% block content %}
+    <div class="col-12 text-center">
+      <div class="error mx-auto" data-text="404">404</div>
+      <p class="lead text-gray-800 mb-5">Página no encontrada</p>
+      <p class="text-gray-500 mb-0">Parece ser un glitch en la Matrix...</p>
+      <a href="{{ url_for('main.home') }}">&larr; Volver al Dashboard</a>
+    </div>
+{% endblock content %}

apiweb/templates/errors/500.html

@@ -0,0 +1,9 @@
+{% extends "layout.html" %}
+{% block content %}
+    <div class="col-12 text-center">
+      <div class="error mx-auto" data-text="500">500</div>
+      <p class="lead text-gray-800 mb-5">Error Interno del Sistema</p>
+      <p class="text-gray-500 mb-0">Ocurrió un error al realizar la operación. Notifique al <b>Administrador del Sistema</b></p>
+      <a href="{{ url_for('main.home') }}">&larr; Volver al Dashboard</a>
+    </div>
+{% endblock content %}

apiweb/templates/layout.html

@@ -0,0 +1,73 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <!-- Required meta tags -->
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+
+    <!-- Bootstrap CSS -->
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
+    <script src="https://kit.fontawesome.com/9797d9c6de.js" crossorigin="anonymous"></script>
+
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='main.css') }}">
+
+    {% if title %}
+        <title>iLab GTFS-RT API - {{ title }}</title>
+    {% else %}
+        <title>iLab GTFS-RT API</title>
+    {% endif %}
+</head>
+<body>
+    <header class="site-header">
+      <nav class="navbar navbar-expand-md navbar-dark bg-steel fixed-top ">
+        <div class="container">
+          <a class="navbar-brand mr-4" href="{{ url_for('main.home') }}">iLab GTFS-RT API</a>
+          <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarToggle" aria-controls="navbarToggle" aria-expanded="false" aria-label="Toggle navigation">
+            <span class="navbar-toggler-icon"></span>
+          </button>
+          <div class="collapse navbar-collapse" id="navbarToggle">
+            <div class="navbar-nav mr-auto">
+
+            <a class="nav-item nav-link" href="{{ url_for('main.home') }}">API</a>
+
+
+            </div>
+            <!-- Navbar Right Side -->
+            <div class="navbar-nav">
+
+            <a class="nav-item nav-link" href="">Información del Navegador</a>
+            <a class="nav-item nav-link" href="">Acerca de...</a>
+
+            </div>
+          </div>
+        </div>
+      </nav>
+    </header>
+    <main role="main" class="container">
+      <div class="row justify-content-lg-center">
+        <div class="col-md-8">
+          {% with messages = get_flashed_messages(with_categories=true) %}
+            {% if messages %}
+              {% for category, message in messages %}
+                <div class="alert alert-{{ category }}">
+                  {{  message }}
+                </div>
+              {% endfor %}
+            {% endif %}
+          {% endwith %}
+          {% block admin %}{% endblock %}
+          {% block content %}{% endblock %}
+        </div>
+
+        </div>
+      </div>
+    </main>
+
+
+    <!-- Optional JavaScript -->
+    <!-- jQuery first, then Popper.js, then Bootstrap JS -->
+    <script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+</body>
+</html>

apiweb/templates/summary.html

@@ -0,0 +1,15 @@
+{% extends "layout.html" %}
+{% block content %}
+<article class="media content-section">
+
+<div class="media-body">
+<legend class="border-bottom mb-4">Últimas 10 mediciones</legend>
+
+{% for record in records %}
+    <p class="article-content"><a href="{{ url_for("main.concepcion", registro_id=record.id) }}">Medicion las {{record.timestamp}}</a></p>
+{% endfor %}
+</div>
+</article>
+<div class="d-flex justify-content-end">
+</div>
+{% endblock content %}

docker-compose.sample.yml

@@ -0,0 +1,63 @@
+version: '3'
+
+services:
+
+  gtfs-ccp-fetcher:
+    build:
+        dockerfile: Dockerfiles/Dockerfile.fetcher
+        context: .
+    image: dev.ilab.cl/tdtp/ccp_gtfs_realtime_fetcher:latest
+    restart: unless-stopped
+    volumes:
+      - ./static:/srv/static
+    environment:
+      - DEBUG=False
+      - GTFS_RT_ENDPOINT=https://datamanager.dtpr.transapp.cl/data/gtfs-rt/concepcion.proto
+      - GTFS_RT_KEY=GTFS_KEY_XXXX_YYYY_ZZZZ
+      - GTFS_RT_INTERVAL=60
+      - SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://db_user:db_pass@db01.tdtp.ilab.cl/db_name
+      - TIMEOUT=1200
+    logging:
+      driver: syslog
+      options:
+        syslog-address: "udp://rsyslog.vpc.ilab.cl:514"
+        tag: "{{.Name}}"
+
+  gtfs-ccp-updater:
+    build:
+        dockerfile: Dockerfiles/Dockerfile.updater
+        context: .
+    image: dev.ilab.cl/tdtp/ccp_gtfs_realtime_updater:latest
+    restart: unless-stopped
+    volumes:
+      - ./static:/srv/static
+    environment:
+      - DEBUG=False
+      - SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://db_user:db_pass@db01.tdtp.ilab.cl/db_name
+      - TIMEOUT=1200
+    logging:
+      driver: syslog
+      options:
+        syslog-address: "udp://rsyslog.vpc.ilab.cl:514"
+        tag: "{{.Name}}"
+
+  gtfs-ccp-api:
+    build:
+        dockerfile: Dockerfiles/Dockerfile.api
+        context: .
+    image: dev.ilab.cl/tdtp/ccp_gtfs_realtime_api:latest
+    restart: unless-stopped
+    volumes:
+      - ./static:/srv/static
+    environment:
+      - DEBUG=False
+      - SECRET_KEY=4d6f45a5fc12445dbac2f59c3b6c7cb2
+      - SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://db_user:db_pass@db01.tdtp.ilab.cl/db_name
+      - TIMEOUT=1200
+    ports:
+      - 4001:8000
+    logging:
+      driver: syslog
+      options:
+        syslog-address: "udp://rsyslog.vpc.ilab.cl:514"
+        tag: "{{.Name}}"

fetch.py

@@ -0,0 +1,8 @@
+# coding: utf-8
+from fetcher.main import sched
+from fetcher.model.feed import init_db, db, engine
+
+
+if __name__ == '__main__':
+    init_db(db, engine)
+    sched.start()

fetcher/main.py

@@ -0,0 +1,45 @@
+from apscheduler.schedulers.background import BlockingScheduler
+from fetcher.model.feed import db, Registros
+from tempfile import NamedTemporaryFile
+from shutil import copyfile
+from datetime import datetime
+import requests
+import logging
+import time
+import sys
+import os
+
+log = logging.getLogger('Fetcher')
+log.setLevel(logging.DEBUG)
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+log.addHandler(handler)
+
+def load_proto():
+    inicio = datetime.now()
+    final_dir  = "static/ccp/{}/{}/{}".format(inicio.strftime("%Y"), inicio.strftime("%m"), inicio.strftime("%d"))
+    final_name = "static/ccp/{}/{}/{}/ccp_gtfs_{}.proto".format(inicio.strftime("%Y"), inicio.strftime("%m"), inicio.strftime("%d"), inicio.strftime("%Y%m%d_%H%M_%S"))
+
+    try:
+        response = requests.get("{}?apikey={}".format(os.environ.get('GTFS_RT_ENDPOINT'), os.environ.get('GTFS_RT_KEY')))
+        response.raise_for_status()
+        with NamedTemporaryFile() as tmp:
+            tmp.write(response.content)
+            os.makedirs(os.path.abspath(final_dir), exist_ok=True)
+            copyfile(tmp.name, os.path.abspath(final_name))
+        nuevo_registro = Registros(filename=final_name, status=0)
+        db.add(nuevo_registro)
+        db.commit()
+        log.info("Fetched GTFS-RT Record {} in {}s".format(inicio.strftime("%Y%m%d_%H%M_%S"), (datetime.now()-inicio).total_seconds()))
+    except:
+        import traceback
+        log.error("Failed to fetch GTFS-RT Record {} in {}s".format(inicio.strftime("%Y%m%d_%H%M_%S"), (datetime.now()-inicio).total_seconds()))
+        log.info('Traceback {}'.format(traceback.format_exc()))
+        nuevo_registro = Registros(filename=final_name, status=100)
+        db.add(nuevo_registro)
+        db.commit()
+
+sched = BlockingScheduler()
+sched.add_job(load_proto, 'interval', seconds=int(os.environ.get('GTFS_RT_INTERVAL'))) #will do the print_t work for every 30 seconds

fetcher/model/feed.py

@@ -0,0 +1,85 @@
+# coding: utf-8
+from sqlalchemy import create_engine, MetaData, Column, ForeignKey, Enum, UniqueConstraint, Integer, String, Text, DateTime, PickleType, BigInteger, Index, Float
+from sqlalchemy.orm import relationship as Relationship, sessionmaker
+from sqlalchemy.sql import func
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.ext.declarative import declarative_base
+import os
+
+if not os.environ.get('SQLALCHEMY_DATABASE_URI'):
+	database_uri = 'sqlite:///test.db'
+#	database_uri = 'postgresql+psycopg2://docker:docker@db/docker'
+else:
+	database_uri = os.environ.get('SQLALCHEMY_DATABASE_URI')
+
+engine = create_engine(database_uri, echo=False)
+Session = sessionmaker(bind=engine, autocommit=False, autoflush=True)
+db = Session()
+
+Base = declarative_base()
+
+def init_db(tdb, engine):
+	try:
+		tdb.query(Vehiculo).first()
+	except:
+		import traceback
+		traceback.format_exc()
+		Base.metadata.create_all(bind=engine)
+		tdb.commit()
+
+class Registros(Base):
+    __tablename__ = 'registros'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = Column(Integer, primary_key=True, autoincrement=True)
+
+    filename  = Column(String) #PATH/ccp/2022/01/01/ccp_gtfs_20220101_246060_00.proto
+    status    = Column(Integer, default=0)
+    timestamp = Column(DateTime(timezone=True), default=func.now())
+
+class Vehiculo(Base):
+    __tablename__ = 'vehicle'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = Column(Integer, primary_key=True, autoincrement=True)
+
+    patente   = Column(String(8))
+    timestamp = Column(BigInteger)
+
+class Viaje(Base):
+    __tablename__ = 'trip'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = Column(Integer, primary_key=True, autoincrement=True)
+
+    trip_id      = Column(String(50))
+    route_id     = Column(Integer())
+    direction_id = Column(Integer())
+
+    start_time   = Column(String(8))
+    start_date   = Column(String(8))
+    timestamp  = Column(BigInteger)
+
+class Entidades(Base):
+    __tablename__ = 'entity'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id         = Column(Integer, primary_key=True, autoincrement=True)
+    entity     = Column(String(50))
+
+    vehicleid  = Column(Integer, ForeignKey('gtfsrt.vehicle.id'), nullable=False)
+    tripid     = Column(Integer, ForeignKey('gtfsrt.trip.id'), nullable=True)
+    timestamp  = Column(BigInteger)
+
+class Posicion(Base):
+    __tablename__ = 'datapoint'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = Column(Integer, primary_key=True, autoincrement=True)
+
+    latitude  = Column(Float)
+    longitude = Column(Float)
+
+    bearing   = Column(Integer())
+    odometer  = Column(Float)
+    speed     = Column(Float)
+
+    vehicleid = Column(Integer, ForeignKey('gtfsrt.vehicle.id'), nullable=False)
+    tripid    = Column(Integer, ForeignKey('gtfsrt.trip.id'), nullable=True)
+
+    timestamp = Column(BigInteger)

fetcher/requeriments.txt

@@ -0,0 +1,5 @@
+SQLAlchemy
+psycopg2
+protobuf
+requests
+apscheduler

sync.py

@@ -0,0 +1,18 @@
+# coding: utf-8
+from updater.model.feed import init_db, db, engine
+from updater.main import sync_db, log
+import time
+
+if __name__ == '__main__':
+    init_db(db, engine)
+    doki = int(time.time()) + 600
+
+    while True:
+        if not sync_db():
+            i = int(time.time())
+            if i >= doki:
+                doki = i + 600
+                log.info('Heartbeat')
+                time.sleep(20)
+        else:
+            doki = int(time.time()) + 600

updater/main.py

@@ -0,0 +1,95 @@
+from updater.model.feed import db, Registros, Vehiculo, Viaje, Entidades, Posicion
+import utils.gtfs_realtime_pb2
+from time import strftime
+from datetime import datetime
+import logging
+import json
+import time
+import sys
+import os
+
+log = logging.getLogger('updater')
+log.setLevel(logging.DEBUG)
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+log.addHandler(handler)
+
+def sync_db():
+    inicio = datetime.now()
+    registrodb = db.query(Registros).filter(Registros.status<5).order_by(Registros.id.asc()).first()
+    if registrodb is None:
+        return False
+
+    log.info("Record {} has status {}".format(registrodb.filename, registrodb.status))
+
+    registrodb.status = registrodb.status + 1
+    db.commit()
+
+    try:
+        feed = utils.gtfs_realtime_pb2.FeedMessage()
+        with open(os.path.abspath(registrodb.filename), 'rb') as file:
+            feed.ParseFromString(file.read())
+
+        for item in feed.entity:
+            patente = item.vehicle.vehicle.license_plate
+            itemts = item.vehicle.timestamp
+
+            patentedb = db.query(Vehiculo).filter(Vehiculo.patente==patente).one_or_none()
+            if patentedb is None:
+                patentedb = Vehiculo(patente=patente, timestamp=itemts)
+                db.add(patentedb)
+            elif patentedb.timestamp < itemts:
+                patentedb.timestamp = itemts
+            db.commit()
+
+            if item.HasField('trip_update'):
+                tripid = item.trip_update.trip.trip_id
+
+                tripdb = db.query(Viaje).filter(Viaje.trip_id==tripid).one_or_none()
+                if tripdb is None:
+                    tripid = item.trip_update.trip.trip_id
+                    routeid = item.trip_update.trip.route_id
+                    direccionid = item.trip_update.trip.direction_id
+                    tstar = item.trip_update.trip.start_time
+                    dstart = item.trip_update.trip.start_date
+
+                    tripdb = Viaje(trip_id=tripid, route_id=routeid, direction_id=direccionid, start_time=tstar, start_date=dstart, timestamp=itemts)
+                    db.add(tripdb)
+                elif tripdb.timestamp < itemts:
+                    tripdb.timestamp = itemts
+                db.commit()
+                tripdbid = tripdb.id
+            else:
+                tripdbid = None
+
+            entidadesdb = db.query(Entidades).filter(Entidades.entity==str(item.id)).one_or_none()
+            if entidadesdb is None:
+                entidadesdb = Entidades(entity=str(item.id), vehicleid=patentedb.id, tripid=tripdbid, timestamp=itemts)
+                db.add(entidadesdb)
+            elif entidadesdb.timestamp < itemts:
+                entidadesdb.timestamp = itemts
+            db.commit()
+
+            posiciondb = db.query(Posicion).filter(Posicion.vehicleid==patentedb.id, Posicion.timestamp==itemts).one_or_none()
+            if posiciondb is None:
+                lat = item.vehicle.position.latitude
+                long = item.vehicle.position.longitude
+                bearing = item.vehicle.position.bearing
+                odometer = item.vehicle.position.odometer
+                speed = item.vehicle.position.speed
+
+                posiciondb = Posicion(vehicleid=patentedb.id, tripid=tripdbid, latitude=lat, longitude=long, bearing=bearing, odometer=odometer, speed=speed, timestamp=itemts)
+                db.add(posiciondb)
+                db.commit()
+
+        registrodb.status=50
+        db.commit()
+        log.info("GTFS-RT {} Ingested in {}s".format(registrodb.filename, (datetime.now()-inicio).total_seconds()))
+        return True
+    except:
+        import traceback
+        log.error("Failed to Ingest Record {} in {}s (attempt: {}/5)".format(registrodb.filename, (datetime.now()-inicio).total_seconds(), registrodb.status))
+        log.info('Traceback {}'.format(traceback.format_exc()))

updater/model/feed.py

@@ -0,0 +1,85 @@
+# coding: utf-8
+from sqlalchemy import create_engine, MetaData, Column, ForeignKey, Enum, UniqueConstraint, Integer, String, Text, DateTime, PickleType, BigInteger, Index, Float
+from sqlalchemy.orm import relationship as Relationship, sessionmaker
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.sql import func
+from sqlalchemy.ext.declarative import declarative_base
+import os
+
+if not os.environ.get('SQLALCHEMY_DATABASE_URI'):
+	database_uri = 'sqlite:///test.db'
+#	database_uri = 'postgresql+psycopg2://docker:docker@db/docker'
+else:
+	database_uri = os.environ.get('SQLALCHEMY_DATABASE_URI')
+
+engine = create_engine(database_uri, echo=False)
+Session = sessionmaker(bind=engine, autocommit=False, autoflush=True)
+db = Session()
+
+Base = declarative_base()
+
+def init_db(tdb, engine):
+	try:
+		tdb.query(Vehiculo).first()
+	except:
+		import traceback
+		traceback.format_exc()
+		Base.metadata.create_all(bind=engine)
+		tdb.commit()
+
+class Registros(Base):
+    __tablename__ = 'registros'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = Column(Integer, primary_key=True, autoincrement=True)
+
+    filename  = Column(String) #PATH/ccp/2022/01/01/ccp_gtfs_20220101_246060_00.proto
+    status    = Column(Integer, default=0)
+    timestamp = Column(DateTime(timezone=True), default=func.now())
+
+class Vehiculo(Base):
+    __tablename__ = 'vehicle'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = Column(Integer, primary_key=True, autoincrement=True)
+
+    patente   = Column(String(8))
+    timestamp = Column(BigInteger)
+
+class Viaje(Base):
+    __tablename__ = 'trip'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = Column(Integer, primary_key=True, autoincrement=True)
+
+    trip_id      = Column(String(50))
+    route_id     = Column(Integer())
+    direction_id = Column(Integer())
+
+    start_time   = Column(String(8))
+    start_date   = Column(String(8))
+    timestamp  = Column(BigInteger)
+
+class Entidades(Base):
+    __tablename__ = 'entity'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id         = Column(Integer, primary_key=True, autoincrement=True)
+    entity     = Column(String(50))
+
+    vehicleid  = Column(Integer, ForeignKey('gtfsrt.vehicle.id'), nullable=False)
+    tripid     = Column(Integer, ForeignKey('gtfsrt.trip.id'), nullable=True)
+    timestamp  = Column(BigInteger)
+
+class Posicion(Base):
+    __tablename__ = 'datapoint'
+    __table_args__ = { 'schema': 'gtfsrt' }
+    id        = Column(Integer, primary_key=True, autoincrement=True)
+
+    latitude  = Column(Float)
+    longitude = Column(Float)
+
+    bearing   = Column(Integer())
+    odometer  = Column(Float)
+    speed     = Column(Float)
+
+    vehicleid = Column(Integer, ForeignKey('gtfsrt.vehicle.id'), nullable=False)
+    tripid    = Column(Integer, ForeignKey('gtfsrt.trip.id'), nullable=True)
+
+    timestamp = Column(BigInteger)

updater/requeriments.txt

@@ -0,0 +1,4 @@
+SQLAlchemy
+psycopg2
+protobuf
+requests

web.py

@@ -0,0 +1,8 @@
+# coding: utf-8
+from apiweb.model.feed import Registros
+from apiweb.main import create_app
+
+iapp = create_app()
+
+if __name__ == '__main__':
+    iapp.run()