Commit 7ae13fe1 authored by guglielmo's avatar guglielmo

Timeout and User-Agent added to the requests, to minimise the errors.

Values are configurable in constance live settings.
parent 3d32be4a
......@@ -290,9 +290,12 @@ CONSTANCE_CONFIG = {
'SMTP_USERNAME': (env("EMAIL_HOST_USER", default=""), "Username for the SMTP host"),
'SMTP_PASSWORD': (env("EMAIL_HOST_PASS", default=""), "Password for the SMTP host"),
'EMAIL_SUBJECT_PREFIX': (env("EMAIL_SUBJECT_PREFIX", default="[servizio verificafonti] "), "Email subject prefix"),
'EMAIL_FROM': (env("DEFAULT_FROM_EMAIL", default="noreply@openpolis.it"), "From email address for notifications")
'EMAIL_FROM': (env("DEFAULT_FROM_EMAIL", default="noreply@openpolis.it"), "From email address for notifications"),
'REQUESTS_MAX_TIMEOUT': (env("RQEUSTS_MAX_TIMEOUT", default=10), "Max timeout for requests in seconds", int),
'REQUESTS_UA': (env("REQUESTS_UA", default=""), "User agent in requests"),
}
CONSTANCE_IGNORE_ADMIN_VERSION_CHECK = True
CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend'
# END CONSTANCE (LIVE SETTINGS) CONFIGURATION
......
import difflib
import datetime
from django.core import management
from django.utils.timezone import now
from taskmanager.logging_utils import LoggingBaseCommand
from project.webapp.models import Content
......@@ -92,7 +92,7 @@ class Command(LoggingBaseCommand):
if options['dryrun'] is False:
content.verification_status = Content.STATUS_ERROR
content.verification_error = err_msg
content.verified_at = datetime.datetime.now()
content.verified_at = now()
content.save()
self.logger.warning("{0}/{1} - {2} while processing {3} (id: {4})".format(
cnt + 1, len(contents), err_msg, content.title, content.id
......
# coding: utf-8
from constance import config
from django.utils.translation import ugettext_lazy as _
from django.db import models
from django.utils import timezone
......@@ -111,7 +112,11 @@ class Content(models.Model):
"""
try:
res = requests.get(self.url, timeout=5)
headers = {}
if config.REQUESTS_UA:
headers['User-Agent'] = config.REQUESTS_UA
res = requests.get(self.url, timeout=config.REQUESTS_MAX_TIMEOUT, headers=headers)
if not res.ok:
raise Exception("Internal Urllib error")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment