.gitignore

@@ -120,5 +120,4 @@ venv.bak/
 *.rdb
 *.sql.gz
 /data
-docker-compose.staging.yml
 ipython-notebooks

.gitlab/.gitlab-ci.yml

@@ -147,8 +147,8 @@ deploy to staging:
     - echo "$CLIENT_CERT" > $DOCKER_CERT_PATH/cert.pem
     - echo "$CLIENT_KEY" > $DOCKER_CERT_PATH/key.pem
     - docker build --compress -t openpolis/opdm/opdm-service:latest .
-    - docker-compose down
-    - docker-compose up -d --build
+    - docker-compose -f docker-compose.yml -f docker-compose.staging.yml down
+    - docker-compose -f docker-compose.yml -f docker-compose.staging.yml up -d --build
     - docker exec opdm-service_web python manage.py makemigrations popolo
     - docker exec opdm-service_web python manage.py migrate popolo
     - docker exec opdm-service_web python manage.py collectstatic --noinput

CHANGELOG.md

@@ -5,7 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](<http://keepachangelog.com/en/1.0.0/>)
 and this project adheres to [Semantic Versioning](<http://semver.org/spec/v2.0.0.html>).
 
-## [unchanged]
+## [1.3.3]
+
+### Added
+Task to import world geometries, with dependencies from various sources;
+Task to import electoral constituencies' geoms for 2018, from ISTAT.
+
+### Changed 
+``django-popolo`` release upgraded to 3.0.2 in requirements.txt
+
+## [1.3.2]
+
+### Changed
+- Migrations reset: the only dependencies from external dependencies can be the __initial migration.
 
 ### Fixed
 - Tests fixed
@@ -568,8 +580,9 @@ in the test stage.
 
 [atoka]: https://atoka.io
 
-[unchanged]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.3.1...master
-[1.3.1]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.30...v1.3.1
+[unchanged]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.3.2...master
+[1.3.1]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.3.1...v1.3.2
+[1.3.1]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.3.0...v1.3.1
 [1.3.0]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.2.3...v1.3.0
 [1.2.3]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.2.2...v1.2.3
 [1.2.2]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.18...v1.2.2

CONTRIBUTORS.txt

 Guglielmo Celata
-Gabriele Lucci
\ No newline at end of file
+Gabriele Lucci
+Paolo Melchiorre

Dockerfile

@@ -33,8 +33,8 @@ COPY requirements.txt /app/
 RUN pip3 install --upgrade pip pip-tools && pip-sync
 
 # check for new changes in django-popolo and reinstall it if necessary
-# ADD "https://gitlab.depp.it/openpolis/django-popolo/commits/master?format=atom" /dev/null
-# RUN pip3 install --exists-action=w -e git+https://gitlab.depp.it/openpolis/django-popolo@master#egg=django-popolo
+ADD "https://gitlab.depp.it/openpolis/django-popolo/commits/master?format=atom" /dev/null
+RUN pip3 install --exists-action=w -e git+https://gitlab.depp.it/openpolis/django-popolo@master#egg=django-popolo
 
 # check for new changes in opdm-etl
 ADD "https://gitlab.depp.it/openpolis/opdm/opdm-etl/commits/master?format=atom" /dev/null

README.md

@@ -29,10 +29,9 @@ redis-server (to store sessions).
 Connections to these servers should be defined in the `.env` file, as
 `DATABASE_URL` and `REDIS_URL` values.
 
-To start developing, clone this repository, get into the `project` directory, 
-then:
+To start developing, clone this repository, then:
 
-    cd web
+    cd config
     
     # generate (and modify) the .env file
     cp .env.sample .env
@@ -53,7 +52,7 @@ then:
     pip install pip-tools
     pip install --upgrade pip
     pip-compile
-    pip install -r requirements.txt
+    pip-sync requirements.txt
 
     # when developing them, re-install packages from local dirs
     pip uninstall django-popolo && pip install -e /Users/gu/Workspace/django-popolo
@@ -163,11 +162,11 @@ When a push to the gitlab.depp.it repository is performed:
 
 To tests the service:
 
-    python manage.py test
+    python manage.py test --settings=config.settings.test
 
 To extract a [``coverage``][coverage.py] report:
 
-    coverage run manage.py test
+    coverage run manage.py test --settings=config.settings.test
     coverage report
 
 

config/.env.sample

@@ -5,7 +5,7 @@
 # ADMIN_NAME=admin
 # DEFAULT_FROM_EMAIL=admin@example.com
 
-DATABASE_URL=postgresql://postgres:@localhost/opdm
+DATABASE_URL=postgis://postgres:@localhost/opdm
 
 # media and static files may be served from totally different directories in production.
 # For example:

config/settings/base.py

@@ -34,13 +34,8 @@ env = environ.Env()
 # Operating System Environment variables have precedence over variables defined
 # in the .env file, that is to say variables from the .env files will only be
 # used if not defined as environment variables.
-READ_DOT_ENV_FILE = env.bool("DJANGO_READ_DOT_ENV_FILE", default=True)
-if READ_DOT_ENV_FILE:
-    dot_env_path = str(CONFIG_PATH / ".env")
-    dot_env_file = Path(dot_env_path)
-    if dot_env_file.is_file():
-        env.read_env(str(dot_env_file))
-
+if env.str("DJANGO_ENV_FILE", default=True):
+    environ.Env.read_env(str(CONFIG_PATH / ".env"))
 
 # GENERAL
 # ------------------------------------------------------------------------------
@@ -74,7 +69,9 @@ ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default="*")
 # DATABASES
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#databases
-DATABASES = {"default": env.db("DATABASE_URL")}
+DATABASES = {
+    "default": env.db("DATABASE_URL")
+}
 DATABASES["default"]["ATOMIC_REQUESTS"] = True
 
 # URLS
@@ -120,12 +117,12 @@ LOCAL_APPS = [
     "project.labels",
     "project.topics",
     "project.audit.apps.AuditConfig",  # Custom app config for easyaudit
+    "project.elections",
 ]
 
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
 
-
 # AUTHENTICATION
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#authentication-backends
@@ -305,7 +302,6 @@ LOGGING = {
     },
 }
 
-
 # CACHES CONFIGURATION
 # -----------------------------------------------------------------------------
 if env.bool("USE_REDIS_CACHE", False):
@@ -352,7 +348,6 @@ DEBUG_JSON_AS_HTML = env("DEBUG_JSON_AS_HTML", default=False)
 if DEBUG_JSON_AS_HTML:
     MIDDLEWARE += ("config.middleware.JsonAsHtml",)
 
-
 # REST FRAMEWORK CONFIGURATION
 # -----------------------------------------------------------------------------
 REST_FRAMEWORK = {
@@ -361,9 +356,7 @@ REST_FRAMEWORK = {
     # - a cookie,
     # - a basic_auth header
     # - a jwt header (json web token)
-    "DEFAULT_PERMISSION_CLASSES": [
-        "api_v1.permissions.OPDMPermission"
-    ],
+    "DEFAULT_PERMISSION_CLASSES": ["api_v1.permissions.OPDMPermission"],
     "DEFAULT_AUTHENTICATION_CLASSES": (
         "rest_framework_simplejwt.authentication.JWTAuthentication",
         "rest_framework.authentication.SessionAuthentication",

docker-compose.staging.yml

+# nginx overrides to deploy on standard staging docker-compose stack
+# use docker-compose -f docker-compose.yml -f docker-compose.staging.yml up -d
+
+version: "3.5"
+
+services:
+  nginx:
+    networks:
+      - default
+      - gw
+  web:
+    networks:
+      - default
+      - gw
+
+networks:
+  default:
+    external: false
+    name: opdm
+  gw:
+    external: true
+

docker-compose.yml

@@ -14,6 +14,7 @@ services:
       - solr:solr
     environment:
       - DATABASE_URL=postgis://${POSTGRES_USER}:${POSTGRES_PASS}@postgres/${POSTGRES_DB}
+      - DJANGO_ENV_FILE=no
       - DEBUG=${DEBUG}
       - SECRET_KEY=${SECRET_KEY}
       - HAYSTACK_SIGNAL_PROCESSOR=project.core.PersonOnlySignalProcessor
@@ -56,7 +57,7 @@ services:
     labels:
       - "traefik.enable=true"
       - "traefik.docker.network=gw"
-      - "traefik.http.routers.opdmsvc.rule=Host(`${DOMAINS}`)"
+      - "traefik.http.routers.opdmsvc.rule=Host(`${DOMAIN}`)"
       - "traefic.http.routers.opdmsvc.entrypoint=websecure"
       - "traefik.http.routers.opdmsvc.tls.certresolver=${CERT_RESOLVER}"
     volumes:

project/__init__.py

@@ -3,7 +3,7 @@ Openpolis Data Manager service package (backend)
 """
 from typing import Optional
 
-__version__ = (1, 3, 1)
+__version__ = (1, 3, 3)
 
 
 def get_version_str() -> str:

project/akas/migrations/0001_initial.py

-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.13 on 2018-07-09 13:40
-from __future__ import unicode_literals
+# Generated by Django 2.2.13 on 2020-10-01 15:51
 
 import django.contrib.postgres.fields.jsonb
 from django.db import migrations, models
@@ -23,10 +21,12 @@ class Migration(migrations.Migration):
                 ('object_id', models.PositiveIntegerField(db_index=True, null=True)),
                 ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='creation time')),
                 ('updated_at', models.DateTimeField(auto_now=True, verbose_name='last modification time')),
-                ('search_params', django.contrib.postgres.fields.jsonb.JSONField(help_text='The search parameters, as query string', unique=True, verbose_name='search parameters')),
-                ('n_similarities', models.PositiveSmallIntegerField(verbose_name='n of similarities')),
-                ('loader_context', django.contrib.postgres.fields.jsonb.JSONField(help_text='a JSON representation of the item and the context it was being imported into', verbose_name='item')),
+                ('search_params', django.contrib.postgres.fields.jsonb.JSONField(help_text='The search parameters, as dictionary', verbose_name='search parameters')),
+                ('n_similarities', models.PositiveSmallIntegerField(blank=True, null=True, verbose_name='n of similarities')),
+                ('loader_context', django.contrib.postgres.fields.jsonb.JSONField(blank=True, help_text='a JSON representation of the item and the context it was being imported into', null=True, verbose_name='loader context')),
                 ('is_resolved', models.BooleanField(default=False, help_text='Whether the aka has been resolved (associated to an object)', verbose_name='is associated')),
+                ('hashed', models.CharField(blank=True, default='', help_text='Hash used to enforce uniqueness of json fields, and avoid postgres error on index too big', max_length=64, null=True, unique=True)),
+                ('data_source_url', models.URLField(blank=True, help_text='The data source URL', max_length=350, null=True, verbose_name='url')),
                 ('content_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
             ],
             options={

project/akas/migrations/0002_auto_20180730_1832.py

-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.13 on 2018-07-30 16:32
-from __future__ import unicode_literals
-
-import django.contrib.postgres.fields.jsonb
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('akas', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='aka',
-            name='loader_context',
-            field=django.contrib.postgres.fields.jsonb.JSONField(help_text='a JSON representation of the item and the context it was being imported into', verbose_name='loader context'),
-        ),
-        migrations.AlterField(
-            model_name='aka',
-            name='search_params',
-            field=django.contrib.postgres.fields.jsonb.JSONField(help_text='The search parameters, as dictionary', verbose_name='search parameters'),
-        ),
-        migrations.AlterUniqueTogether(
-            name='aka',
-            unique_together=set([('search_params', 'loader_context')]),
-        ),
-    ]

project/akas/migrations/0003_auto_20181015_1251.py

-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.15 on 2018-10-15 10:51
-from __future__ import unicode_literals
-
-import hashlib
-import json
-
-from django.db import migrations, models
-
-# def compute_hashes(apps, schema_editor):
-#     # We can't import the Person model directly as it may be a newer
-#     # version than this migration expects. We use the historical version.
-#     AKA = apps.get_model('akas', 'AKA')
-#     n_akas = AKA.objects.count()
-#     print("")
-#     print("computing and storing hashes for existing AKAs")
-#     for n, a in enumerate(AKA.objects.all(), 1):
-#         text = json.dumps(a.search_params) + json.dumps(a.loader_context)
-#         hashed = hashlib.sha256(text.encode('utf-8')).hexdigest()
-#         a.hashed = hashed
-#         a.save()
-#         if n%1000 == 0:
-#             print("{0}/{1}".format(n, n_akas))
-#
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('akas', '0002_auto_20180730_1832'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='aka',
-            name='hashed',
-            field=models.CharField(
-                default='',
-                help_text='Hash used to enforce uniqueness of json fields, and avoid postgres error on index too big',
-                max_length=64, null=True
-            ),
-        ),
-        migrations.AlterUniqueTogether(
-            name='aka',
-            unique_together=set([]),
-        ),
-        # migrations.RunPython(compute_hashes),
-    ]

project/akas/migrations/0004_auto_20181015_1303.py

-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.15 on 2018-10-15 10:51
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('akas', '0003_auto_20181015_1251'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='aka',
-            name='hashed',
-            field=models.CharField(default='', help_text='Hash used to enforce uniqueness of json fields, and avoid postgres error on index too big', max_length=64, unique=True),
-        ),
-    ]

project/akas/migrations/0005_aka_data_source_url.py

-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.20 on 2019-05-03 17:31
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('akas', '0004_auto_20181015_1303'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='aka',
-            name='data_source_url',
-            field=models.URLField(blank=True, help_text='The data source URL', max_length=350, null=True, verbose_name='url'),
-        ),
-    ]

project/akas/migrations/0006_auto_20190822_1729.py

-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.20 on 2019-08-22 15:29
-from __future__ import unicode_literals
-
-import django.contrib.postgres.fields.jsonb
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('akas', '0005_aka_data_source_url'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='aka',
-            name='hashed',
-            field=models.CharField(blank=True, default='', help_text='Hash used to enforce uniqueness of json fields, and avoid postgres error on index too big', max_length=64, null=True, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='aka',
-            name='loader_context',
-            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, help_text='a JSON representation of the item and the context it was being imported into', null=True, verbose_name='loader context'),
-        ),
-        migrations.AlterField(
-            model_name='aka',
-            name='n_similarities',
-            field=models.PositiveSmallIntegerField(blank=True, null=True, verbose_name='n of similarities'),
-        ),
-    ]

project/api_v1/routers.py

@@ -29,6 +29,13 @@ from project.api_v1.views.viewsets import (
     ClassificationViewSet,
     OrganizationRelationshipViewSet)
 
+from elections.views import (
+    ElectoralCandidateResultViewSet,
+    ElectoralListResultViewSet,
+    ElectoralResultViewSet,
+    PoliticalColourViewSet
+)
+
 
 class CustomAPIv1Router(DefaultRouter):
     """
@@ -127,5 +134,13 @@ def register(router_class: Type[SimpleRouter], **initkwargs) -> SimpleRouter:
     router.register(prefix=r"keyevents", viewset=KeyEventViewSet)
     # /classifications
     router.register(prefix=r"classifications", viewset=ClassificationViewSet)
+    # /political_colours
+    router.register(r"political_colours", PoliticalColourViewSet)
+    # /electoral_results
+    router.register(r"electoral_results", ElectoralResultViewSet)
+    # /list_results
+    router.register(r"list_results", ElectoralListResultViewSet)
+    # /candidate_results
+    router.register(r"candidate_results", ElectoralCandidateResultViewSet)
 
     return router

project/atoka/migrations/0001_initial.py

-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.15 on 2019-02-06 16:26
-from __future__ import unicode_literals
+# Generated by Django 2.2.13 on 2020-10-01 15:51
 
 import django.core.validators
 from django.db import migrations, models
@@ -12,7 +10,7 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('popolo', '0016_auto_20190117_0900'),
+        ('popolo', '0001_initial'),
     ]
 
     operations = [
@@ -22,7 +20,7 @@ class Migration(migrations.Migration):
                 ('organization', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='economics', serialize=False, to='popolo.Organization')),
                 ('employees', models.PositiveIntegerField(blank=True, help_text='Latest number of employees', null=True)),
                 ('revenue', models.BigIntegerField(blank=True, help_text='Latest yearly revenue', null=True)),
-                ('revenue_trend', models.FloatField(blank=True, help_text='Latest trend in revenues', null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(2019)])),
+                ('revenue_trend', models.FloatField(blank=True, help_text='Latest trend in revenues', null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(100.0)])),
                 ('capital_stock', models.BigIntegerField(blank=True, help_text='Latest capital stock', null=True)),
                 ('assets', models.BigIntegerField(blank=True, help_text='Latest yearly assets', null=True)),
                 ('costs', models.BigIntegerField(blank=True, help_text='Latest yearly costs', null=True)),
@@ -36,16 +34,17 @@ class Migration(migrations.Migration):
                 ('services_and_tp_goods_charges', models.BigIntegerField(blank=True, help_text='Latest yearly service and tp goods charges', null=True)),
                 ('staff_costs', models.BigIntegerField(blank=True, help_text='Latest yearly staff costs', null=True)),
                 ('is_public', models.BooleanField(default=False, help_text='If the organization is listed in the publick stock exchange')),
+                ('pub_part_percentage', models.PositiveIntegerField(blank=True, help_text='Percentage of shares held or referrable to public institutions', null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(100)])),
             ],
         ),
         migrations.CreateModel(
             name='OrganizationEconomicsHistorical',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('year', models.PositiveIntegerField(help_text='Year of validity of economics indicator', validators=[django.core.validators.MinValueValidator(2000), django.core.validators.MaxValueValidator(2019)])),
+                ('year', models.PositiveIntegerField(help_text='Year of validity of economics indicator', validators=[django.core.validators.MinValueValidator(1980), django.core.validators.MaxValueValidator(2020)])),
                 ('employees', models.PositiveIntegerField(blank=True, help_text="Current year's number of employees", null=True)),
                 ('revenue', models.BigIntegerField(blank=True, help_text='Revenue for current year', null=True)),
-                ('revenue_trend', models.FloatField(blank=True, help_text="Current year's revenue trend with respect to previous year", null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(2019)])),
+                ('revenue_trend', models.FloatField(blank=True, help_text="Current year's revenue trend with respect to previous year", null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(100.0)])),
                 ('capital_stock', models.BigIntegerField(blank=True, help_text="Current year's capital stock", null=True)),
                 ('assets', models.BigIntegerField(blank=True, help_text="Current year's assets", null=True)),
                 ('costs', models.BigIntegerField(blank=True, help_text="Current year's costs", null=True)),

project/atoka/migrations/0002_auto_20190214_1250.py

-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.15 on 2019-02-14 11:50
-from __future__ import unicode_literals
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('atoka', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='organizationeconomics',
-            name='pub_part_percentage',
-            field=models.PositiveIntegerField(blank=True, help_text='Percentage of shares held or referrable to public institutions', null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(100)]),
-        ),
-        migrations.AlterField(
-            model_name='organizationeconomics',
-            name='revenue_trend',
-            field=models.FloatField(blank=True, help_text='Latest trend in revenues', null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(100.0)]),
-        ),
-        migrations.AlterField(
-            model_name='organizationeconomicshistorical',
-            name='revenue_trend',
-            field=models.FloatField(blank=True, help_text="Current year's revenue trend with respect to previous year", null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(100.0)]),
-        ),
-        migrations.AlterField(
-            model_name='organizationeconomicshistorical',
-            name='year',
-            field=models.PositiveIntegerField(help_text='Year of validity of economics indicator', validators=[django.core.validators.MinValueValidator(1980), django.core.validators.MaxValueValidator(2019)]),
-        ),
-    ]