Related
I'm trying to integrate logging with loki in my django app like this:
handler = logging_loki.LokiHandler(
url="http://localhost:3100/loki/api/v1/push",
tags={"app": "django", "env": ENV},
version="1",
)
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '[%(asctime)s] {%(module)s} [%(levelname)s] - %(message)s',
'datefmt': '%d-%m-%Y %H:%M:%S'
},
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'standard',
}
},
'loggers': {
'': {
'handlers': ['console', handler], # this doesnt work
'level': 'DEBUG',
'propagate': True,
# 'name': ENV
}
}
}
What do I need to change so that django uses this custom handler? I tried just referencing the handler object in the logging dict but that didn't need to be the right approach.
I also tried this:
LOGGING_CONFIG = None
logging.config.dictConfig(LOGGING)
logging.getLogger(__name__).addHandler(handler)
but that's not sending any logs to loki
Try this:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '[%(asctime)s] {%(module)s} [%(levelname)s] - %(message)s',
'datefmt': '%d-%m-%Y %H:%M:%S'
},
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'standard',
},
'loki': {
'level': 'INFO',
'class': 'logging_loki.LokiHandler',
'url': "http://localhost:3100/loki/api/v1/push",
'tags' {"app": "django", "env": ENV},
'version': "1",
},
},
'loggers': {
'': {
'handlers': ['console', 'loki'],
'level': 'DEBUG',
'propagate': True,
}
}
}
Specifically, you have to:
Define you handler in the handlers dictionary in the LOGGING setting. The configuration here is used to specify the initialisation arguments for the handler.
Then assign this handler using its key (loki in the example I've given above) to the logger(s) you want it to handle.
I'm using Sentry with Django like this:
sentry_integration = DjangoIntegration()
sentry_sdk.init(
dsn="https://xxx#sentry.io/xxx",
integrations=[sentry_integration]
)
and with these settings for logging:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'class': 'logging.StreamHandler',
},
'file': {
'class': 'logging.FileHandler',
'filename': os.path.join(BASE_DIR, 'logs', 'django.log'),
},
},
'loggers': {
'django': {
'handlers': ['file', 'console'],
'level': 'DEBUG',
},
'django.template': {
'handlers': ['file', 'console'],
'level': 'INFO',
},
'app': {
'handlers': ['file', 'console'],
'level': 'DEBUG',
}
},
}
If I instiantiate and call a logger in my code, this gets sent to Sentry.
import logging
logger = logging.getLogger(__name__)
logger.error("error!")
However, now I'd like to also log .warning calls. The documentation says to do this:
sentry_logging = LoggingIntegration(event_level=logging.WARNING)
sentry_sdk.init(
dsn="https://xxx#sentry.io/xxx",
integrations=[sentry_logging]
)
But the LoggingIntegration is used rather than the DjangoIntegration. I've tried to use DjangoIntegration in the code above but I get this error:
TypeError: init() got an unexpected keyword argument 'event_level'
Is this possible?
The integrations argument is a list. You can pass in both integrations:
init(
integrations=[LoggingIntegration(...), DjangoIntegration(...)]
)
I've tried the below code for logging in django settings
from google.cloud import logging as google_cloud_logging
log_client = google_cloud_logging.Client()
log_client.setup_logging()
LOGGING = {
'version': 1,
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
},
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler'
},
'stackdriver_logging': {
'class': 'google.cloud.logging.handlers.CloudLoggingHandler',
'client': log_client
},
},
'loggers': {
'django': {
'handlers': ['stackdriver_logging'],
'level': 'INFO',
'propagate': True,
},
'django.request': {
'handlers': [
'stackdriver_logging',
'mail_admins'
],
'level': 'ERROR',
}
},
}
gunicorn.conf.py
bind = "127.0.0.1:8000"
workers = 3
loglevel = "debug"
proc_name = "hhh"
daemon = False
I was able write to stackdriver logging with django development server but logging not working when I use gunicorn. But django.request is working.
Problem: logger "django" not being called and showing no errors with gunicorn
Please help.
After trying many ways finally it worked with below configuration
gcp_utils/stackdriver_logging.py
import logging
import traceback
from django.views.debug import ExceptionReporter
from google.cloud import error_reporting, logging as google_cloud_logging
client = error_reporting.Client()
log_client = google_cloud_logging.Client()
log_client.setup_logging()
class StackdriverErrorHandler(logging.Handler):
def __init__(self):
logging.Handler.__init__(self)
def emit(self, record):
try:
request = record.request
if request.user.is_authenticated():
user = request.user.email
else:
user = 'Anonymous'
status_code = getattr(record, 'status_code', None)
http_context = error_reporting.HTTPContext(
method=request.method,
url=request.build_absolute_uri(),
user_agent=request.META.get('HTTP_USER_AGENT'),
referrer=request.META.get('HTTP_REFERER'),
response_status_code=status_code,
remote_ip=request.META.get('REMOTE_ADDR')
)
except Exception as e:
print e
http_context = None
if record.exc_info:
exc_info = record.exc_info
else:
exc_info = (None, record.getMessage(), None)
reporter = ExceptionReporter(request, is_email=True, *exc_info)
tb_frames = reporter.get_traceback_frames()
if tb_frames:
tb = tb_frames[-1]
else:
tb = {}
report_location = {
'filePath': tb.get('filename'),
'lineNumber': tb.get('lineno'),
'functionName': tb.get('function')
}
client._send_error_report(
message=traceback.format_exc(),
http_context=http_context,
user=user,
report_location=report_location
)
settings.py
import sys
from google.cloud import logging as google_cloud_logging
log_client = google_cloud_logging.Client()
if not DEBUG:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'stream': sys.stdout,
},
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler'
},
'stackdriver_logging': {
'class': 'google.cloud.logging.handlers.CloudLoggingHandler',
'client': log_client
},
'stackdriver_error_reporting': {
'level': 'ERROR',
'class': 'gcp_utils.stackdriver_logging.StackdriverErrorHandler',
}
},
'loggers': {
'django': {
'handlers': ['console', 'stackdriver_logging'],
'level': 'DEBUG',
'propagate': True,
},
'django.request': {
'handlers': [
'stackdriver_logging',
'mail_admins'
],
'level': 'ERROR',
}
},
}
gunicorn.conf.py
from google.cloud import logging as google_cloud_logging
log_client = google_cloud_logging.Client()
log_client.setup_logging()
bind = "127.0.0.1:8000"
workers = 3
loglevel = "debug"
proc_name = "django_app"
daemon = False
pythonpath = "/path/to/python/"
timeout = 90
accesslog = '/home/user/logs/debug.log'
logconfig_dict = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler'
},
'stackdriver_logging': {
'class': 'google.cloud.logging.handlers.CloudLoggingHandler',
'client': log_client
},
},
'loggers': {
'gunicorn.access': {
'handlers': ['stackdriver_logging'],
'level': 'INFO',
'propagate': True,
},
'django.request': {
'handlers': [
'stackdriver_logging',
'mail_admins'
],
'level': 'ERROR',
}
},
}
running gunicorn process
$ gunicorn -c gunicorn.conf.py wsgi:application
We must define the logconfig_dict in-order to make it work with google stackdriver logging.
I'm trying to figure out logging in Django (checking out Python logging itself for the first time too for that matter).
I have the following in my settings.py
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'file': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': '/home/afzalsh/logs/debug.log',
},
},
'loggers': {
'django.request': {
'handlers': ['file'],
'level': 'DEBUG',
'propagate': True,
},
},
}
This is the exact code as what Django documentation has as the first example.
Then I have a log message in a class based view.
import logging
.
.
.
class StationHome(View):
.
.
.
def post(self,request, pk):
station = Station.objects.get(pk=pk)
form = StationForm(request.POST, instance=station)
if form.is_valid():
logger = logging.getLogger(__name__)
logger.error("Yep!")
form.save()
return HttpResponseRedirect(reverse("home_station",
kwargs={'pk':pk},
)
)
else:
return HttpResponse("Form Invalid")
.
.
.
The debug.log file has got created but it doesn't get populated with any message on going through the particular code. The expected process do happen.
You've requested the logger associated with __name__, so you get one named the same as the current Python module. However, the only one you've actually configured is "django.request". You should use that in your call to getLogger instead.
In your settings your handler dict contains level 'debug' for file where as in your code you are using 'error' level to log.
logger.error("Yep!")
Try using 'debug' level in your code. Something like this:
logger.debug("Yep!")
also posting my settings here so that you can use that as a reference.
{
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'standard' : {
'format' : "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
'datefmt' : "%d/%b/%Y %H:%M:%S"
},
},
'handlers': {
'console':{
'level': 'DEBUG',
'class': 'logging.StreamHandler'
},
'null' : {
'level':'DEBUG',
'class':'django.utils.log.NullHandler',
},
'file' : {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': 'debug.log',
},
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler'
}
},
'loggers': {
'django.request': {
'handlers': ['mail_admins'],
'level': 'ERROR',
'propagate': True,
},
'django': {
'handlers': ['file'],
'propogate': True,
'level': 'DEBUG',
},
'matchroom_app':{
'handlers' : ['file'],
'level': 'DEBUG',
},
'matchroom_post':{
'handlers' : ['file'],
'level' : 'DEBUG',
},
'matchroom_social':{
'handlers': ['file'],
'level' : 'DEBUG',
},
}
}
My Django 1.3 logging setup causes all log messages to be output twice. I've read that importing settings.py two or more times has caused this problem in the past, but Django 1.3 has new logging features, and I don't think I'm importing settings.py twice anywhere.
settings.py config:
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format': '%(levelname)s: %(message)s'
},
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'simple',
},
},
'loggers': {
'custom': {
'handlers': ['console'],
'level': 'DEBUG',
},
}
}
Code:
import logging
log = logging.getLogger('custom')
log.debug('message1')
log.debug('message2')
Output:
DEBUG: message1
DEBUG:custom:message1
Thanks for your help.
Have you tried setting propagate = False? Along with disable_existing_loggers = True?
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'simple': {
'format': '%(levelname)s: %(message)s'
},
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'simple',
},
},
'loggers': {
'custom': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': False,
},
}
}
You might want to try 'disable_existing_loggers': True?
I've been suffered by the same issue. And I fixed it by redirecting the root logs to a file, and project logs to both the file and the console.
I grep my code, and could not find anywhere basicConfig() exists, also tried to set disable_existing_loggers to True, It doesn't help, finally solved the problem by set a file logger. I guess it maybe a problem by design in some cases.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
# exact format is not important, this is the minimum information
'format': '%(asctime)s %(name)-12s %(lineno)d %(levelname)-8s %(message)s',
},
},
'handlers': {
'console': {
#'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'simple',
},
# Add Handler for mail_admins for `warning` and above
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler',
},
'file': {
#'level': 'INFO',
'class': 'logging.FileHandler',
'formatter': 'simple',
'filename': os.path.join(os.path.dirname(PROJECT_ROOT), 'crawler.admin.log'),
},
},
'loggers': {
# root logger
'': {
'level': 'INFO',
'handlers': ['file', 'mail_admins'],
},
'scrapy': {
'level': 'WARNING',
'handlers': ['console', 'mail_admins'],
'propagate': False,
},
'crawleradmin': {
'level': 'INFO',
'handlers': ['console', 'file', 'mail_admins'],
# required to avoid double logging with root logger
'propagate': False,
},
},
}