Verified Commit fe7f34ec authored by guglielmo's avatar guglielmo

django-tenants removed; management commands adapted to work with django-uwsgi-taskmanager

parent aee6738b
......@@ -82,11 +82,8 @@ DEFAULT_FROM_EMAIL = env("DEFAULT_FROM_EMAIL", default=ADMIN_EMAIL)
# DATABASES
# ------------------------------------------------------------------------------
# https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {"default": env.db("DATABASE_URL", engine='django_tenants.postgresql_backend')}
DATABASES = {"default": env.db("DATABASE_URL")}
DATABASES["default"]["ATOMIC_REQUESTS"] = True
DATABASE_ROUTERS = (
'django_tenants.routers.TenantSyncRouter',
)
# MEDIA CONFIGURATION
# -----------------------------------------------------------------------------
......@@ -134,7 +131,6 @@ TEMPLATES = [
"OPTIONS": {
"debug": DEBUG,
"loaders": [
"django_tenants.template.loaders.filesystem.Loader", # Must be first
"django.template.loaders.filesystem.Loader",
"django.template.loaders.app_directories.Loader",
],
......@@ -154,15 +150,11 @@ TEMPLATES = [
}
]
MULTITENANT_TEMPLATE_DIRS = [
os.path.join(str(PROJECT_PATH), "tenants", "%s", "templates"),
]
# MIDDLEWARE CONFIGURATION
# -----------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#middleware-classes
MIDDLEWARE = [
'django_tenants.middleware.main.TenantMainMiddleware',
"django.middleware.common.CommonMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
......@@ -179,9 +171,8 @@ ROOT_URLCONF = "config.urls"
# APPS CONFIGURATION
# -----------------------------------------------------------------------------
SHARED_APPS = (
DJANGO_APPS = (
# Default Django apps:
"django_tenants",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
......@@ -191,26 +182,15 @@ SHARED_APPS = (
# Admin panel and documentation:
"django.contrib.admin",
# 'django.contrib.admindocs',
# admin enhancers apps
"django_object_actions",
"django_admin_row_actions",
# Django helper
"django_extensions",
# Customers or tenants
"project.tenants",
)
# Apps specific for this project go here.
TENANT_APPS = (
# Default Django apps:
"django.contrib.contenttypes",
"django.contrib.auth",
"django.contrib.sessions",
"django.contrib.sites",
"django.contrib.messages",
"django.contrib.staticfiles",
# Admin panel and documentation:
"django.contrib.admin",
# admin enhancers apps
"django_object_actions",
"django_admin_row_actions",
LOCAL_APPS = (
# The app
"project.webapp",
# Live settings
......@@ -223,7 +203,7 @@ TENANT_APPS = (
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = list(SHARED_APPS) + [app for app in TENANT_APPS if app not in SHARED_APPS]
INSTALLED_APPS = DJANGO_APPS + LOCAL_APPS
# AUTHENTICATION CONFIGURATION
# -----------------------------------------------------------------------------
......@@ -239,20 +219,11 @@ LOGS_PATH = env("LOGS_PATH", default=os.path.normpath(str(RESOURCES_PATH) + "/lo
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"filters": {
"tenant_context": {
"()": "django_tenants.log.TenantContextFilter"
},
},
"formatters": {
"verbose": {
"format": "[%(asctime)s] %(levelname)s [%(pathname)s:%(funcName)s:%(lineno)s] %(message)s",
"datefmt": "%d/%b/%Y %H:%M:%S",
},
"tenant_context": {
"format": "[%(schema_name)s:%(domain_url)s] "
"%(levelname)-7s %(asctime)s %(message)s",
},
"simple": {"format": "[%(asctime)s] %(levelname)s %(message)s", "datefmt": "%d/%b/%Y %H:%M:%S"},
},
"handlers": {
......@@ -262,13 +233,11 @@ LOGGING = {
"filename": os.path.normpath(os.path.join(LOGS_PATH, "opsv.log")),
"maxBytes": 1024 * 1024 * 10, # 10 MB
"backupCount": 7,
"filters": ["tenant_context", ]
},
"console": {
"level": "DEBUG",
"class": "logging.StreamHandler",
"stream": sys.stdout,
"filters": ["tenant_context", ]
},
},
"loggers": {
......@@ -310,14 +279,8 @@ WSGI_APPLICATION = "wsgi.application"
TEST_RUNNER = "django.test.runner.DiscoverRunner"
# END TESTING CONFIGURATION
# TENANT CONFIGURATION
# -------------------
TENANT_MODEL = "tenants.Instance"
TENANT_DOMAIN_MODEL = "tenants.Domain"
# END TENANT CONFIGURATION
# CONSTANCE (LIVE SETTINGS) CONFIGURATION
# -------------------
# -----------------------------------------------------------------------------
CONSTANCE_CONFIG = {
'USE_SLACK': (False, "Enable notifications to slack channel. If False, then email is used", bool),
'SLACK_TOKEN': ("", "The token for slack integration, as read from slack"),
......@@ -328,6 +291,15 @@ CONSTANCE_IGNORE_ADMIN_VERSION_CHECK = True
CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend'
# END CONSTANCE (LIVE SETTINGS) CONFIGURATION
# TASK_MANAGER CONFIGURATION
# -----------------------------------------------------------------------------
TASK_MANAGER_N_REPORTS_INLINE = 5
TASK_MANAGER_N_LINES_IN_REPORT_INLINE = 10
TASK_MANAGER_SHOW_LOGVIEWER_LINK = True
TASK_MANAGER_USE_FILTER_COLLAPSE = True
# END TASK_MANAGER CONFIGURATION
VERSION = __import__(PROJECT_PACKAGE).__version__
# Usually set by the Gitlab runner, to set it manually:
......
......@@ -12,3 +12,13 @@ Copy ``config/sample/.env`` into ``config/``, and change values according to you
$ createdb -Upostgres DB_NAME
$ python project/manage.py migrate
$ python project/manage.py runserver
To launch an uWSGI server with a process to handle asynchronous tasks:
.. code-block:: bash
uwsgi --check-static=./resources --http=:8000 --master \
--module=wsgi --callable=application \
--pythonpath=/Users/gu/Workspace/op-sources-verification \
--processes=1 --spooler=./resources/uwsgi-spooler --spooler-processes=1
\ No newline at end of file
from django.contrib import admin
from django.contrib.sites.models import Site
from django_tenants.admin import TenantAdminMixin
from .models import Instance, Domain
class DomainInline(admin.TabularInline):
"""An inline for related domains."""
model = Domain
extra = 0
fields = ('domain', )
max_num = 3
@admin.register(Instance)
class InstanceAdmin(TenantAdminMixin, admin.ModelAdmin):
inlines = [DomainInline, ]
list_display = ('name', '_domains', 'on_trial')
def _domains(self, obj):
return list(obj.domains.values_list('domain', flat=True))
_domains.short_description = 'Domains'
def has_module_permission(self, request):
"""
Return empty perms dict thus hiding the model from admin index.
"""
if 'localhost' not in list(request.tenant.domains.values_list('domain', flat=True)):
return {}
else:
return super().has_module_permission(request)
def has_add_permission(self, request):
"""
Return empty perms dict blocking users from instance application from adding instances
"""
if 'localhost' not in list(request.tenant.domains.values_list('domain', flat=True)):
return {}
else:
return super().has_add_permission(request)
def get_queryset(self, request):
if 'localhost' not in list(request.tenant.domains.values_list('domain', flat=True)):
return Instance.objects.none()
# return super().get_queryset(request).filter(id__in=[request.tenant.id])
else:
return super().get_queryset(request)
admin.site.unregister(Site)
from django.apps import AppConfig
class TenantsConfig(AppConfig):
name = 'project.tenants'
# Generated by Django 2.2.1 on 2019-05-31 08:59
from django.db import migrations, models
import django.db.models.deletion
import django_tenants.postgresql_backend.base
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Instance',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('schema_name', models.CharField(db_index=True, max_length=63, unique=True, validators=[django_tenants.postgresql_backend.base._check_schema_name])),
('name', models.CharField(max_length=100)),
('on_trial', models.BooleanField(default=True)),
('created_on', models.DateField(auto_now_add=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Domain',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('domain', models.CharField(db_index=True, max_length=253, unique=True)),
('is_primary', models.BooleanField(db_index=True, default=True)),
('tenant', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='domains', to='tenants.Instance')),
],
options={
'abstract': False,
},
),
]
from django.db import models
from django_tenants.models import TenantMixin, DomainMixin
class Instance(TenantMixin):
name = models.CharField(max_length=100)
on_trial = models.BooleanField(default=True)
created_on = models.DateField(auto_now_add=True)
# default true, schema will be automatically created and synced when it is saved
auto_create_schema = True
def __str__(self):
return self.name
class Domain(DomainMixin):
def __str__(self):
return f"{self.domain}"
{% extends django_slack %}
{% block text %}
Cambiamenti ed errori nei siti sotto controllo
*Dominio*: {{ domain }}
*Cambiamenti*
{% for content in modified_contents %}
• <http://{{domain}}/sitescheck/content/{{content.id}}|{{ content.title }}> (<http://{{domain}}/diff/{{content.id}}"|diff>)
{% endfor %}
*Errori*
{% for content in failed_contents %}
• <http://{{domain}}/sitescheck/content/{{content.id}}|{{ content.title }}> (<http://{{domain}}/diff/{{content.id}}"|diff>)
{% endfor %}
{% endblock %}
{% block endpoint_url %}{{ config.SLACK_ENDPOINT_URL }}{% endblock %}
{% block channel %}{{ config.SLACK_CHANNEL }}{% endblock %}
{% block username %}{{ config.SLACK_USERNAME }}{% endblock %}
{% block token %}{{ config.SLACK_TOKEN }}{% endblock %}
{% extends django_slack %}
{% block text %}
Notifica task: Una prova
{% endblock %}
{% block endpoint_url %}{{ config.SLACK_ENDPOINT_URL }}{% endblock %}
{% block channel %}{{ config.SLACK_CHANNEL }}{% endblock %}
{% block username %}{{ config.SLACK_USERNAME }}{% endblock %}
{% block token %}{{ config.SLACK_TOKEN }}{% endblock %}
from django.core.management.base import BaseCommand
# coding=utf-8
from taskmanager.logging_utils import LoggingBaseCommand
from project.webapp.models import Content
class Command(BaseCommand):
class Command(LoggingBaseCommand):
help = 'List all contents in the DB ID, title and verification status'
def handle(self, **options):
def handle(self, *args, **options):
"""Contains the command logic.
Launch the scrapy crawl histadmin subprocess and log the output.
:param args:
:param options:
:return:
"""
self.setup_logger(__name__, formatter_key="simple", **options)
for content in Content.objects.all():
print("{0.id} - \"{0.title}\" ({0.verification_status})".format(content))
self.logger.info("{0.id} - \"{0.title}\" ({0.verification_status})".format(content))
from django.core.management.base import BaseCommand
from taskmanager.logging_utils import LoggingBaseCommand
from project.webapp.models import Content
import datetime
class Command(BaseCommand):
class Command(LoggingBaseCommand):
help = """
Get live content,
from specified URI's ids, or from all URIs.
......@@ -45,6 +45,8 @@ class Command(BaseCommand):
)
def handle(self, *args, **options):
self.setup_logger(__name__, formatter_key="simple", **options)
offset = options['offset']
limit = options['limit']
......@@ -58,14 +60,14 @@ class Command(BaseCommand):
contents = Content.objects.filter(id__in=args)
if len(contents) == 0:
print("no content to get this time")
self.logger.info("no content to get this time")
for cnt, content in enumerate(contents):
err_msg = ''
try:
content.update()
except IOError:
err_msg = "Errore: Url non leggibile: %s" % content.url
err_msg = "Url non leggibile: %s" % content.url
except Exception as e:
err_msg = "Errore sconosciuto: {0}".format(e)
finally:
......@@ -75,7 +77,7 @@ class Command(BaseCommand):
content.verification_error = err_msg
content.verified_at = datetime.datetime.now()
content.save()
print("{0}/{1} - {2} while processing {3} (id: {4})".format(
self.logger.warning("{0}/{1} - {2} while processing {3} (id: {4})".format(
cnt + 1, len(contents), err_msg, content.title, content.id
))
else:
......@@ -86,6 +88,6 @@ class Command(BaseCommand):
)
)
if options['showhtml'] is True:
print("Meaningful content: {0}".format(content.get_live_content()))
self.logger.info("Contenuto significativo: {0}".format(content.get_live_content()))
if options['dryrun'] is False:
content.save()
import difflib
import datetime
from django.core.management.base import BaseCommand
from taskmanager.logging_utils import LoggingBaseCommand
from project.webapp.models import Content
class Command(BaseCommand):
class Command(LoggingBaseCommand):
help = "Verify content of specified URI's ids or all"
def add_arguments(self, parser):
......@@ -50,8 +50,9 @@ class Command(BaseCommand):
)
def handle(self, *args, **options):
offset = options['offset']
self.setup_logger(__name__, formatter_key="simple", **options)
offset = options['offset']
limit = options['limit']
if len(args) == 0:
......@@ -64,14 +65,14 @@ class Command(BaseCommand):
contents = Content.objects.filter(id__in=args)
if len(contents) == 0:
print("no content to check this time")
self.logger.info("no content to check this time")
for cnt, content in enumerate(contents):
err_msg = ''
try:
_ = content.verify(options['dryrun'])
except IOError:
err_msg = "Errore: Url non leggibile: {0}".format(content.url)
err_msg = "Url non leggibile: {0}".format(content.url)
except Exception as e:
err_msg = "Errore sconosciuto: {0}".format(e)
finally:
......@@ -81,7 +82,7 @@ class Command(BaseCommand):
content.verification_error = err_msg
content.verified_at = datetime.datetime.now()
content.save()
print("{0}/{1} - {2} while processing {3} (id: {4})".format(
self.logger.warning("{0}/{1} - {2} while processing {3} (id: {4})".format(
cnt + 1, len(contents), err_msg, content.title, content.id
))
else:
......@@ -93,9 +94,9 @@ class Command(BaseCommand):
)
)
if options['showmeat'] is True:
print("Meaningful content: {0}".format(content.get_live_content()))
self.logger.info("Contenuto significativo: {0}".format(content.get_live_content()))
if options['showdiff'] is True:
live = content.get_live_content().splitlines(1)
stored = content.meat.splitlines(1)
diff = difflib.ndiff(live, stored)
print("".join(diff))
self.logger.info("".join(diff))
import slack
from django.contrib.sites.models import Site
from django.core.management.base import BaseCommand
from django.core.mail import EmailMultiAlternatives
from django.conf import settings
from constance import config
from taskmanager.logging_utils import LoggingBaseCommand
from project.webapp.models import Content, Recipient
class Command(BaseCommand):
class Command(LoggingBaseCommand):
help = 'Notify variations, if necessary, to all recipients'
def add_arguments(self, parser):
......@@ -25,12 +25,6 @@ class Command(BaseCommand):
default="slack",
help='How notifications are sent.'
)
parser.add_argument(
'--slack-engine',
dest='slack_engine',
default="RequestsBackend",
help='The slack engine to be used: use ConsoleBackend to test.'
)
def handle(self, *args, **options):
"""
......@@ -39,7 +33,10 @@ class Command(BaseCommand):
:param options:
:return:
"""
self.setup_logger(__name__, formatter_key="simple", **options)
context = {
'config': config,
'domain': Site.objects.get_current().domain,
'n_analysed_contents': Content.objects.count(),
'modified_contents': Content.objects.filter(verification_status=Content.STATUS_CHANGED),
......@@ -48,28 +45,29 @@ class Command(BaseCommand):
'n_failed_contents': Content.objects.filter(verification_status=Content.STATUS_ERROR).count(),
}
summary_txt = \
f"Trovati {context['n_modified_contents']} siti con variazioni e " \
f"{context['n_failed_contents']} con errori."
if options['notification_method'] == 'slack':
self.handle_slack(context=context, **options)
self.handle_slack(context=context, summary_txt=summary_txt, **options)
else:
self.handle_email(context=context, **options)
self.handle_email(context=context, summary_txt=summary_txt, **options)
@staticmethod
def handle_slack(**kwargs):
def handle_slack(self, **kwargs):
"""
:param kwargs:
:return:
"""
context = kwargs.get('context')
context['config'] = config
summary_txt = \
f"Trovati {context['n_modified_contents']} siti con variazioni e " \
f"{context['n_failed_contents']} con errori."
summary_txt = kwargs.get('summary_txt', '')
linked_txt = \
f"Trovati *{context['n_modified_contents']}* " \
f"<http://opdm.opsv.io:8000/admin/webapp/content/?verification_status__exact=1|siti con variazioni> e " \
f"<http://{context['domain']}/admin/webapp/content/?verification_status__exact=1|siti con variazioni> e " \
f"*{context['n_failed_contents']}* " \
f"<http://opdm.opsv.io:8000/admin/webapp/content/?verification_status__exact=2|siti con errori>."
f"<http://{context['domain']}/admin/webapp/content/?verification_status__exact=2|siti con errori>."
msg_blocks = [
{
......@@ -88,7 +86,6 @@ class Command(BaseCommand):
"text": linked_txt
}
},
]
client = slack.WebClient(token=config.SLACK_TOKEN)
......@@ -97,53 +94,56 @@ class Command(BaseCommand):
text=summary_txt,
blocks=msg_blocks
)
self.logger.info(summary_txt)
@staticmethod
def handle_email(**kwargs):
def handle_email(self, **kwargs):
"""
:param kwargs:
:return:
"""
context = kwargs.get('context')
domain = context['domain']
modified_contents = context['modified_contents']
failed_contents = context['failed_contents']
msg_txt = ""
msg_html = ""
if modified_contents.count():
msg_txt += "Cambiamenti:\n"
msg_html += "Questo l'elenco dei siti cambiati: <br/><ul " \
"style=\"list-style-type:none\">"
for content in modified_contents:
msg_txt += " - {0.title}\n".format(content)
msg_html += """
<li>
<a href="http://{1}/sitescheck/content/{0.id}">
{0.title}
</a>
- <a href="http://{1}/diff/{0.id}">diff</a>
</li>
""".format(content, domain)
msg_html += "</ul>"
if failed_contents.count():
msg_txt += "Errori:\n"
msg_html += "Questo l'elenco dei siti con errori: <br/><ul style=\"list-style-type:none\">"
for content in failed_contents:
msg_txt += " - {0.title}\n".format(content)
msg_html += """
<li><a href="{0.url}">{0.title}</a></li>
""".format(content)
msg_html += "</ul>"
if msg_txt != '':
# if modified_contents.count():
# msg_txt += "Cambiamenti:\n"
# msg_html += "Questo l'elenco dei siti cambiati: <br/><ul " \
# "style=\"list-style-type:none\">"
# for content in modified_contents:
# msg_txt += " - {0.title}\n".format(content)
# msg_html += """
# <li>
# <a href="http://{1}/sitescheck/content/{0.id}">
# {0.title}
# </a>
# - <a href="http://{1}/diff/{0.id}">diff</a>
# </li>
# """.format(content, domain)
# msg_html += "</ul>"
#
# if failed_contents.count():
# msg_txt += "Errori:\n"
# msg_html += "Questo l'elenco dei siti con errori: <br/><ul style=\"list-style-type:none\">"
# for content in failed_contents:
# msg_txt += " - {0.title}\n".format(content)
# msg_html += """
# <li><a href="{0.url}">{0.title}</a></li>
# """.format(content)
# msg_html += "</ul>"
#
if context['n_modified_contents'] + context['n_failed_contents'] > 0:
msg_txt = kwargs.get('summary_txt')
msg_html = \
f"Trovati " \
f"<a href=\"http://{context['domain']}/admin/webapp/content/?verification_status__exact=1\">" \
f"{context['n_modified_contents']} siti con variazioni </a> e " \
f"<a href=\"http://{context['domain']}/admin/webapp/content/?verification_status__exact=2\">" \
f"{context['n_failed_contents']} siti con errori</a>."
recipients = []
for recipient in Recipient.objects.all():
recipients.append(recipient.email)
print("recipients: " + "; ".join(recipients))
self.logger.info("Sending emails to: " + "; ".join(recipients))
if kwargs.get('dryrun', False) is False:
try:
......@@ -151,10 +151,10 @@ class Command(BaseCommand):
msg = EmailMultiAlternatives(subject, msg_txt, settings.DEFAULT_FROM_EMAIL, recipients)
msg.attach_alternative(msg_html, "text/html")
msg.send()
print("ok")
self.logger.info("Emails sent.")
except Exception as e:
print("error sending email: {0}".format(e))
self.logger.error("error sending email: {0}".format(e))
else:
print("will not send a bit; it's a dry run!")
self.logger.debug("will not send a bit; it's a dry run!")
else:
print("No changes detected!")
self.logger.info("No changes detected. Smile!")
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
import slack
from django.conf import settings
from django.db import connection
from django_tenants.utils import get_tenant_model, get_public_schema_name
from constance import config
......@@ -12,25 +9,11 @@ from project.webapp.models import Content
class Command(BaseCommand):
help = 'Test slack message'
def get_tenant(self, schema='public'):
"""Return the Tenant for the given schema"""
TenantModel = get_tenant_model()
all_tenants = TenantModel.objects.all()
if not all_tenants:
raise CommandError("""There are no tenants in the system.
To learn how create a tenant, see:
https://django-tenants.readthedocs.org/en/latest/use.html#creating-a-tenant""")
return TenantModel.objects.get(schema_name=schema)
def handle(self, *args, **options):
@slack.RTMClient.run_on(event='message')
def say_hello(**payload):
connection.set_tenant(self.get_tenant(schema='opdm'))
data = payload['data']
web_client = payload['web_client']
rtm_client = payload['rtm_client']
......@@ -48,4 +31,4 @@ class Command(BaseCommand):
slack_token = config.SLACK_TOKEN
rtm_client = slack.RTMClient(token=slack_token)
rtm_client.start()
print("I'm listening")
\ No newline at end of file
print("I'm listening")
from unittest.mock import patch
from django_tenants.test.cases import TenantTestCase
from django_tenants.test.client import TenantClient
from django.utils import timezone
from django.test import TestCase, Client
from project.webapp.models import Content, OrganisationType, APIException
from project.webapp.tests import html_content, parsed_content