import gevent.monkey gevent.monkey.patch_all() # noqa # place it under the main gevent patch import grpc.experimental.gevent grpc.experimental.gevent.init_gevent() # noqa import json import time from openai import OpenAI import time from threading import Thread import schedule import sentry_sdk from sentry_sdk.integrations.redis import RedisIntegration from errors.errors import BadRequestCausedAlgorithmException, BadRequestException from flask import Flask, Response from prometheus_client import ( multiprocess, generate_latest, CollectorRegistry, CONTENT_TYPE_LATEST, ) from tasks.celery_app import app from config import config from utils.logging import logging from svc.sample_data import refresh_six_cols_history_data, refresh_two_cols_history_data flask_app = Flask(__name__) sentry_sdk.init( dsn=config.SENTRY_DSN, integrations=[ RedisIntegration(), ], environment=config.ENVIRONMENT, enable_tracing=False, traces_sample_rate=0, ignore_errors=[BadRequestCausedAlgorithmException, BadRequestException], ) @flask_app.get("/metrics") def metrics(): # for customized metrics, not used right now registry = CollectorRegistry() multiprocess.MultiProcessCollector(registry) data = generate_latest(registry) status = '200 OK' headers = [ ('Content-type', CONTENT_TYPE_LATEST), ('Content-Length', str(len(data))) ] return Response(data, status, headers) @flask_app.get("/healthz") def healthz(): return "I'm OK", 200 def run_metrics_server(): logging.info("metrics server start...") flask_app.run(host="0.0.0.0", port=11000) def refresh_financial_data(): logging.info("refresh_financial_data start...") try: refresh_two_cols_history_data("unrate") refresh_two_cols_history_data("fedfunds") refresh_six_cols_history_data("csi300") refresh_six_cols_history_data("gold") refresh_six_cols_history_data("crude_oil") refresh_six_cols_history_data("nasdaq") refresh_six_cols_history_data("dji") except Exception as e: logging.error("refresh financial data failed. error = %s", str(e)) # bug fix def run_cronjob(): try: if config.DISABLE_CRONJOB: logging.info("cronjob disabled, skip...") return logging.info("run_cronjob start...") from tasks.eda import mark_expired_jobs, send_measure_metrics schedule.every(1).minutes.do(mark_expired_jobs) schedule.every(5).minutes.do(send_measure_metrics) if config.ENVIRONMENT == "prod": schedule.every(1).day.at("00:00").do(refresh_financial_data) while True: schedule.run_pending() time.sleep(1) except Exception as e: logging.error("cronjob failed. error = %s", str(e)) if __name__ == "__main__": argv = [ 'worker', '--loglevel=INFO', "-P", "gevent", "-Ofair", ] Thread(target=run_metrics_server).start() Thread(target=run_cronjob).start() app.worker_main(argv)