.dockerignore

@@ -14,3 +14,4 @@ docker-compose.yml
 docker-compose.*.yml
 Dockerfile
 dump*
+*sql.gz

.gitignore

@@ -118,22 +118,6 @@ venv.bak/
 .nfs*
 
 *.rdb
-
+*.sql.gz
 /data
-resources/data/bdap_anagrafe_enti.csv
-resources/data/parsers
-resources/data/out
-resources/data/atoka
-resources/data/*.json
-resources/uwsgi-spooler
-/resources/data/discrepancies.csv
-resources/data/role_types_atoka_opdm.csv
-/resources/data/cache
 docker-compose.*.yml
-/resources/data/cf_partecipate_mef.csv
-/resources/data/cf_partecipate_opdm.csv
-/resources/data/partecipazioni_mef_amm_centrali.csv
-/resources/data/partecipazioni_mef.csv
-/resources/data/partecipazioni_mef_amm_centrali_not_in_opdm.csv
-/resources/data/partecipazioni_mef_regioni.csv
-/resources/data/partecipazioni_mef_regioni_not_in_opdm.csv

.gitlab-ci.yml → .gitlab/.gitlab-ci.yml

@@ -29,7 +29,7 @@ variables:
     - mdillon/postgis:9.6-alpine
   variables:
     DATABASE_URL: "postgis://${POSTGRES_USER}:${POSTGRES_PASS}@mdillon__postgis/${POSTGRES_DB}"
-    DJANGO_SETTINGS_MODULE: "config.settings"
+    DJANGO_SETTINGS_MODULE: "config.settings.test"
   script:
     - apt-get update
     - apt-get install -y --allow-unauthenticated --no-install-recommends binutils libproj-dev gdal-bin
@@ -49,7 +49,7 @@ build virtualenv:
     - pip install -U pip virtualenv safety
     - virtualenv venv                   # Create virtual environment
     - source venv/bin/activate          # Activate virtual environment
-    - pip install pip-tools
+    - pip install "pip<19.1" pip-tools
     - pip-sync requirements.txt         # Install project requirements inside venv
     - pip freeze                        # List installed packages in venv
   except:
@@ -71,9 +71,26 @@ django tests:
   image: python:3.6-stretch
   <<: *pull_cache_job
   <<: *geodjango_tests
+  retry: 2
   except:
     - schedules
 
+.pages:
+  image: alpine
+  stage: deploy
+  script:
+  - apk --no-cache add py-pip python-dev
+  - pip install sphinx sphinx-rtd-theme sphinx-django-command alabaster
+  - apk --no-cache add make
+  - cd docs
+  - make html
+  - mv _build/html/ ../public/
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
+
 # Deploy: deploy to staging job
 deploy to staging:
   image: tmaier/docker-compose:18.06
@@ -86,6 +103,7 @@ deploy to staging:
     DOCKER_CERT_PATH: "certs"
     DOMAINS: ${DOMAINS_STAGING}
     USE_EMAIL_SMTP: "off"
+    COMPOSE_PROJECT_NAME: "opdm-service"
   script:
     - mkdir $DOCKER_CERT_PATH
     - echo "$CA" > $DOCKER_CERT_PATH/ca.pem
@@ -94,10 +112,10 @@ deploy to staging:
     - docker build --compress -t openpolis/opdm/opdm-service:latest .
     - docker-compose down
     - docker-compose up -d --build
-    - docker exec opdm-service_web_1 python manage.py makemigrations popolo
-    - docker exec opdm-service_web_1 python manage.py migrate popolo
-    - docker exec opdm-service_web_1 python manage.py collectstatic --noinput
-    - docker exec opdm-service_web_1 python manage.py migrate
+    - docker exec opdm-service_web python manage.py makemigrations popolo
+    - docker exec opdm-service_web python manage.py migrate popolo
+    - docker exec opdm-service_web python manage.py collectstatic --noinput
+    - docker exec opdm-service_web python manage.py migrate
     - rm -rf $DOCKER_CERT_PATH
   only:
     - master
@@ -118,18 +136,19 @@ deploy to production:
     DOCKER_HOST: "tcp://${PRODUCTION_HOST_IP}:2376"
     DOCKER_CERT_PATH: "certs"
     DOMAINS: ${DOMAINS_PRODUCTION}
+    COMPOSE_PROJECT_NAME: "opdm-service"
   script:
     - mkdir $DOCKER_CERT_PATH
     - echo "$CA" > $DOCKER_CERT_PATH/ca.pem
     - echo "$CLIENT_CERT" > $DOCKER_CERT_PATH/cert.pem
     - echo "$CLIENT_KEY" > $DOCKER_CERT_PATH/key.pem
-    - docker build -t openpolis/opdm/opdm-service:latest .
+    - docker build --compress -t openpolis/opdm/opdm-service:latest .
     - docker-compose down
     - docker-compose up -d --build --force-recreate
-    - docker exec opdm-service_web_1 python manage.py makemigrations popolo
-    - docker exec opdm-service_web_1 python manage.py migrate popolo
-    - docker exec opdm-service_web_1 python manage.py migrate
-    - docker exec opdm-service_web_1 python manage.py collectstatic --noinput
+    - docker exec opdm-service_web python manage.py makemigrations popolo
+    - docker exec opdm-service_web python manage.py migrate popolo
+    - docker exec opdm-service_web python manage.py migrate
+    - docker exec opdm-service_web python manage.py collectstatic --noinput
     - rm -rf $DOCKER_CERT_PATH
   only:
     - master
@@ -158,7 +177,7 @@ reset_staging_db:on-schedule:
     - echo "$CA" > $DOCKER_CERT_PATH/ca.pem
     - echo "$CLIENT_CERT" > $DOCKER_CERT_PATH/cert.pem
     - echo "$CLIENT_KEY" > $DOCKER_CERT_PATH/key.pem
-    - docker exec -u postgres opdm-service_postgres_1 psql -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'opdm';"
-    - docker exec -u postgres opdm-service_postgres_1 bash -c "cd ~/data && dropdb opdm && createdb opdm && psql opdm < opdm_staging_20181114.sql"
+    - docker exec -u postgres opdm-service_postgres psql -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'opdm';"
+    - docker exec -u postgres opdm-service_postgres bash -c "cd ~/data && dropdb opdm && createdb opdm && psql opdm < opdm_staging_20181114.sql"
   only:
     - schedules

.tmuxinator.yml

@@ -7,7 +7,7 @@ root: ./
 # socket_name: foo
 
 # Ensure used daemons are used before start
-pre: pg_ctl -D /usr/local/var/postgres start 
+on_project_start: if [ $(pg_ctl -D /usr/local/var/postgres status | grep 'no server running' | wc -l) == 1 ]; then pg_ctl -D /usr/local/var/postgres start; fi 
 
 # Runs in each window and pane before window/pane specific commands. Useful for setting up interpreter versions.
 # pre_window: rbenv shell 2.0.0-p247
@@ -25,7 +25,7 @@ pre: pg_ctl -D /usr/local/var/postgres start
 # attach: false
 
 # Runs after everything. Use it to attach to tmux with custom options etc.
-post: pg_ctl -D /usr/local/var/postgres stop
+on_project_exit: if [ $(tmux ls| wc -l) == 0 ]; then pg_ctl -D /usr/local/var/postgres stop; else echo there are $(tmux ls | wc -l) tmux sessions running, could not stop postgres; fi
 
 windows: 
   - shell_plus:

CHANGELOG.md

@@ -2,11 +2,34 @@
 
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
-and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+The format is based on [Keep a Changelog](<http://keepachangelog.com/en/1.0.0/>)
+and this project adheres to [Semantic Versioning](<http://semver.org/spec/v2.0.0.html>).
 
 ## Unreleased
 
+## [1.1.16]
+
+### Added
+
+- meta management task to launch all of ATOKA-related ETL operations
+- ``opdm_ids_file`` parameter added to ``import_atoka_persons`` script
+  to only fetch data for given persons
+- Script to generate personal relationships in neo4j implemented as
+  management task
+- Built-in audit log feature (based on [easyaudit]())
+
+### Changed
+- django-uwsgi-taskmanager upgraded o 2.0.4 (management tasks return result)
+- script that merge duplicates organizations adjusted
+- parameters added to customise `script_check_discrepancies`;
+- urls to affected entities added in log messages;
+
+## [1.1.15]
+
+### Changed
+- import_orgs_from_json refactored, to use JsonDiff class;
+- import_ownerships_from_json refactored, to use JsonDiff class;
+  
 ## [1.1.14]
 
 ### Added
@@ -19,7 +42,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - The script to merge organizations now merge the memberships, before 
   removing 'old' organizations.   
 
-## [1.1.13]
+## 1.1.13 (YANKED)
 
 ### Added
 
@@ -30,7 +53,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - `v1/memberships` now accepts PATCH request, enabling the (partial) update of a whole batch of 
   `Membership objects.
 - Add [`Interval` validator](./project/api_v1/validators.py) to be used when serializing models with
-  `start_date` and `end_date` (`Dateframeable).
+  `start_date` and `end_date` (`Dateframeable`).
 
 ### Changed
 
@@ -73,7 +96,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
-- ATECO classifications for organizations are imported from [[atoka]]
+- ATECO classifications for organizations are imported from [atoka]
 
 ## [1.1.7]
 
@@ -160,7 +183,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - contexts may be specified for [atoka] transformations 
   in import_atoka_transform management task, so that now the
   complete [atoka] import procedure described in the gitlab snippet
-  (https://gitlab.depp.it/openpolis/opdm/opdm-service/snippets/26), 
+  (<https://gitlab.depp.it/openpolis/opdm/opdm-service/snippets/26>), 
   can be launched
 
 ## [1.0.2]
@@ -368,8 +391,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
     - memberships
     - posts
 
-## [0.4.1]
-
 ### Changed
 - Local Administration import refactored, so that the locations
   are fetched from Area, not requested to openpolis api
@@ -447,7 +468,9 @@ in the test stage.
 
 
 [atoka]: https://atoka.io
-[1.1.13]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.11...v1.1.13
+[1.1.16]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.15...v1.1.16
+[1.1.15]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.14...v1.1.15
+[1.1.14]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.12...v1.1.14
 [1.1.12]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.11...v1.1.12
 [1.1.11]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.10...v1.1.11
 [1.1.10]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.9...v1.1.10

DATA_CHANGELOG.md

@@ -5,23 +5,23 @@ All notable changes to the data in OPDM will be documented here.
 
 The format is lousely based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
-## [Unreleased]
+## Unreleased
 
 - administrative history of an organ from MININT:
   - memberships' end date and reason
   - electoral lists
 
-## [2019-02-12]
+## 2019-02-12
 
 - public organization's ownerships (and owned organizations) from ATOKA
 - memberships into organizations imported from ATOKA
    
-## [2019-02-08]
+## 2019-02-08
 
 - minister's dipartimenti and equivalent organizations
 
 
-## [2019-01-09]
+## 2019-01-09
 
 - person's other_names synced from similarities 
 

Dockerfile

@@ -9,7 +9,6 @@ RUN apt-get -qy update \
     && apt-get install -qqqy --no-install-recommends \
         gcc \
         git \
-        libspatialite5 \
         locales \
         python-dev \
         libxml2-dev libxslt-dev \
@@ -41,10 +40,6 @@ RUN pip3 install --exists-action=w -e git+https://gitlab.depp.it/openpolis/djang
 ADD "https://gitlab.depp.it/openpolis/opdm/opdm-etl/commits/master?format=atom" /dev/null
 RUN pip3 install --exists-action=w -e git+https://gitlab.depp.it/openpolis/opdm/opdm-etl.git@master#egg=opdm-etl
 
-# check for new changes in op-task-manager-project
-ADD "https://gitlab.depp.it/openpolis/op-task-manager-project/commits/dev-opdm?format=atom" /dev/null
-RUN pip3 install --exists-action=w -e git+https://gitlab.depp.it/openpolis/op-task-manager-project.git@dev-opdm#egg=django-uwsgi-taskmanager
-
 # remove gcc and build dependencies to keep image small
 RUN apt-get purge -y --auto-remove gcc python-dev
 

README.md

 # Openpolis DataManager Service
 
-[![pipeline status](https://gitlab.depp.it/openpolis/opdm/opdm-service/badges/develop/pipeline.svg)](https://gitlab.depp.it/openpolis/opdm/opdm-service/commits/develop)
-[![coverage report](https://gitlab.depp.it/openpolis/opdm/opdm-service/badges/develop/coverage.svg)](https://gitlab.depp.it/openpolis/opdm/opdm-service/commits/develop)
-
 The Openpolis DataManager service fetch interesting data around 
 the world, mixes and matches it and provides it back to all 
 those in need of it.
@@ -42,28 +39,29 @@ then:
     
     # install and configure pyenv
 
-    # install python 3.6.5 version, using pyenv
+    # install python 3.6.x version, using pyenv
     pyenv install 3.6.5
 
     # define python 3.6.5 as local, in the directory
     pyenv local 3.6.5
 
     # create and activate a virtualenv
-    python -m venv venv
+    virtualenv venv
     source venv/bin/activate
     
     # install requirements
     pip install pip-tools
     pip install --upgrade pip
-    ./install-requirements.sh
+    pip-compile
+    pip install -r requirements.txt
 
-    # install local packages (overwrite)
-    pip install -e /Users/gu/Workspace/django-popolo
-    pip install -e /Users/gu/Workspace/opdm-etl
-    pip install -e /Users/gu/Workspace/op-task-manager-project
+    # when developing them, re-install packages from local dirs
+    pip uninstall django-popolo && pip install -e /Users/gu/Workspace/django-popolo
+    pip uninstall opdm-etl && pip install -e /Users/gu/Workspace/opdm-etl
+    pip uninstall op-task-manager-project && pip install -e /Users/gu/Workspace/op-task-manager-project
     
-    # install other useful packages (docs-writing, ipython, jupyter ...)
-    pip install Sphinx sphinx_rtd_theme ipython jupyter
+    # install other useful packages for devlopers (docs-writing, ipython, jupyter ...)
+    pip install Sphinx sphinx_rtd_theme ipython jupyter safety 
 
     # create database
     createdb -Upostgres opdm
@@ -156,7 +154,7 @@ https://www.capside.com/labs/deploying-full-django-stack-with-docker-compose/
 
 When a push to the gitlab.depp.it repository is performed:
 
-- tests[^1] are performed on the runner
+- tests [^1] are performed on the runner
 - if tests succed and branch is `develop`, deploy latest revision to [`staging`][opdm-service environments] using docker compose
 - if tests succed and branch is `master`, deploy latest revision to [`production`][opdm-service environments] using docker compose
 
@@ -180,6 +178,9 @@ See authors in CONTRIBUTORS.txt
 
 
 
-[^1]: unittest, [flake8](http://flake8.pycqa.org/) for common errors and PEP8 compliancy, [safety](https://pyup.io/safety/) for security checks by pyup
+[^1]: Unittests,
+    [flake8](http://flake8.pycqa.org/) for common errors and PEP8 compliancy,  
+    [safety](https://pyup.io/safety/) for security checks by pyup
+
 [opdm-service environments]: https://gitlab.depp.it/openpolis/opdm/opdm-service/environments
 [coverage.py]: https://coverage.readthedocs.io

compose/.gitignore

+.linkurious_env

compose/backup_opdm.sh

+#!/bin/bash
+
+# data di oggi
+DATE=$(date +%Y%m%d)
+
+curl https://cronitor.link/I2bGKr/run -m 10 || true
+
+# backup db to archive on postgres container
+echo Generating backup archive
+/usr/bin/docker exec -eDATE="$DATE" -upostgres opdm-service_postgres bash -c "cd ~/data; pg_dump opdm > opdm_$DATE.sql; gzip opdm_$DATE.sql; exit"
+
+# copy archive from container on local machine
+echo Copy archive from container
+/usr/bin/docker cp "opdm-service_postgres:/var/lib/postgresql/data/opdm_$DATE.sql.gz" .
+
+# store backup from local  machine to aws S3
+echo Store archive to s3
+/usr/bin/aws --profile s3 s3 cp "opdm_$DATE.sql.gz s3://opdm-service-data/"
+
+# monthly dump
+DAY=$(date +%d)
+if [[ $DAY = 01 ]]; then
+  echo Copying monthly dump
+  /usr/bin/aws --profile s3 s3 cp "s3://opdm-service-data/opdm_$DATE.sql.gz" s3://opdm-service-data/monthly/
+fi
+
+# removing daily backups older than a week
+echo Removing backups older than a week
+for (( i=10; i>=7; i-- ));
+do
+    echo Removing "s3://opdm-service-data/opdm_$(date -d "-$i days" +"%Y%m%d").sql.gz"
+    /usr/bin/aws/ --profile s3 s3 rm "s3://opdm-service-data/opdm_$(date -d "-$i days" +"%Y%m%d").sql.gz"
+done;
+
+# remove archive locally
+rm "opdm_$DATE.sql.gz"
+
+# remove archive on docker-container
+/usr/bin/docker exec -eDATE="$DATE" -upostgres opdm-service_postgres bash -c "rm /var/lib/postgresql/data/opdm_$DATE.sql.gz"
+
+curl https://cronitor.link/I2bGKr/complete -m 10 || true
+

compose/postgres/extract_neo4j_csv.sh

+echo csv_extraction
+  cat extract_neo4j_csv.sql | psql -Uopdm opdm
+
+echo csv_transformation
+rm *_header.csv
+for F in $(ls -1 *.csv | sed -e 's/\.csv$//')
+do
+  echo $F
+  head -n 1 $F.csv > ${F}_header.csv
+  tail -n +2 $F.csv > ${F}_body.csv
+  mv ${F}_body.csv $F.csv
+done
+

compose/postgres/extract_neo4j_csv.sql

+drop function if exists date_2_ts(text);
+
+create or replace function date_2_ts(d text) returns bigint
+    language plpgsql
+as
+$$
+BEGIN
+RETURN date_part('epoch', to_timestamp(d, 'YYYY-MM-DD'));
+END;
+$$;
+
+
+-- areas.csv
+CREATE TEMP VIEW av as
+SELECT
+  a.id as "id:ID(Area-ID)",
+  a.name,
+  a.identifier, a.istat_classification, a.classification,
+  a.inhabitants as "inhabitants:int",
+  a.start_date as "start_date:date",
+  date_2_ts(a.start_date) as "start_date_ts:long",
+  a.end_date as "end_date:date",
+  date_2_ts(a.end_date) as "end_date_ts:long",
+  CASE WHEN a.end_date > current_date::text or a.end_date is null THEN 'true'
+       ELSE 'false'
+  END as "is_current:boolean",
+  'Area'::text as ":LABEL"
+FROM popolo_area as a
+;
+\COPY (SELECT * FROM av) TO '/var/lib/postgresql/neo4j_import/nodes_areas.csv' WITH CSV HEADER
+
+
+-- organizations.csv
+CREATE TEMP VIEW ov as
+SELECT
+  o.id as "id:ID(Organization-ID)",
+  o.name,
+  o.classification,
+  o.founding_date as "founding_date:date",
+  date_2_ts(o.founding_date) as "founding_date_ts:long",
+  o.dissolution_date as "dissolution_date:date",
+  date_2_ts(o.dissolution_date) as "dissolution_date_ts:long",
+  CASE WHEN o.dissolution_date > current_date::text or o.dissolution_date is null THEN 'true'
+       ELSE 'false'
+  END as "is_current:boolean",
+  i.identifier as cf_piva,
+  oe.revenue, oe.employees,
+  'Organizzazione'::text as ":LABEL"
+FROM popolo_organization as o
+LEFT JOIN popolo_identifier i on i.object_id=o.id and i.content_type_id=28 and i.scheme='CF'
+LEFT JOIN atoka_organizationeconomics as oe on oe.organization_id = o.id
+;
+
+\COPY (SELECT * FROM ov) TO '/var/lib/postgresql/neo4j_import/nodes_organizations.csv' WITH CSV HEADER
+
+
+-- persons.csv
+CREATE TEMP VIEW pv as
+SELECT
+  p.id as "id:ID(Person-ID)",
+  p.family_name, p.given_name,
+  p.name, p.image,
+  p.birth_date as "birth_date:date",
+  date_2_ts(p.birth_date) as "birth_date_ts:long",
+  p.death_date as "death_date:date",
+  date_2_ts(p.death_date) as "death_date_ts:long",
+  p.birth_location as birth_location_str,
+  p.birth_location_area_id,
+  CASE WHEN p.death_date is null THEN 'true'
+       ELSE 'false'
+  END as "is_alive:boolean",
+  e.name as education_level,
+  pr.name as profession,
+   'Persona'::text as ":LABEL"
+FROM popolo_person p
+  LEFT join popolo_educationlevel e on e.id=p.education_level_id
+  LEFT join popolo_originalprofession pr on pr.id=p.original_profession_id
+;
+\COPY (SELECT * FROM pv) TO '/var/lib/postgresql/neo4j_import/nodes_persons.csv' WITH CSV HEADER
+
+
+-- areas_parents.csv
+CREATE TEMP VIEW apv as
+SELECT
+  md5(id::text || ':' || parent_id::text)::uuid as "id",
+  id as ":START_ID(Area-ID)",
+  parent_id as ":END_ID(Area-ID)",
+  'PARTE_DI'::text as ":TYPE"
+from popolo_area
+where parent_id is not null
+;
+\COPY (SELECT * FROM apv) TO '/var/lib/postgresql/neo4j_import/rels_areas_parents.csv' WITH CSV HEADER
+
+
+-- organizations_parents.csv
+CREATE TEMP VIEW opv as
+SELECT
+  md5(id::text || ':' || parent_id::text)::uuid as "id",
+  id as ":START_ID(Organization-ID)",
+  parent_id as ":END_ID(Organization-ID)",
+  'PARTE_DI'::text as ":TYPE"
+from popolo_organization
+where parent_id is not null
+;
+\COPY (SELECT * FROM opv) TO '/var/lib/postgresql/neo4j_import/rels_organizations_parents.csv' WITH CSV HEADER
+
+
+-- organizations_areas_parents.csv
+CREATE TEMP VIEW oapv as
+SELECT
+  md5(id::text || ':' || area_id::text)::uuid as "id",
+  id as ":START_ID(Organization-ID)",
+  area_id as ":END_ID(Area-ID)",
+  'ATTIVA_IN'::text as ":TYPE"
+from popolo_organization
+where area_id is not null
+;
+\COPY (SELECT * FROM oapv) TO '/var/lib/postgresql/neo4j_import/rels_organizations_areas_parents.csv' WITH CSV HEADER
+
+
+-- memberships.csv
+CREATE TEMP VIEW mv as
+SELECT
+  md5(person_id::text || ':' || organization_id::text || ':' || label || ':' || coalesce(start_date, 'NULL')::text)::uuid as "id",
+  person_id as ":START_ID(Person-ID)",
+  organization_id as ":END_ID(Organization-ID)",
+  role,
+  label,
+  start_date as "start_date:date",
+  date_2_ts(start_date) as "start_date_ts:long",
+  end_date as "end_date:date",
+  date_2_ts(end_date) as "end_date_ts:long",
+  end_reason,
+  CASE WHEN end_date > current_date::text or end_date is null THEN 'true'
+       ELSE 'false'
+  END as "is_current:boolean",
+  'INCARICO_IN'::text as ":TYPE"
+from popolo_membership
+;
+\COPY (SELECT * FROM mv) TO '/var/lib/postgresql/neo4j_import/rels_memberships.csv' WITH CSV HEADER
+
+
+-- personal_ownerships.csv
+CREATE TEMP VIEW pov as
+SELECT
+  md5(owner_person_id::text || ':' || owned_organization_id::text || ':' || coalesce(start_date, 'NULL')::text)::uuid as "id",
+  owner_person_id as ":START_ID(Person-ID)",
+  owned_organization_id as ":END_ID(Organization-ID)",
+  percentage,
+  start_date as "start_date:date",
+  date_2_ts(start_date) as "start_date_ts:long",
+  end_date as "end_date:date",
+  date_2_ts(end_date) as "end_date_ts:long",
+  end_reason,
+  CASE WHEN end_date > current_date::text or end_date is null THEN 'true'
+       ELSE 'false'
+  END as "is_current:boolean",
+  'DETIENE_QUOTA_DI'::text as ":TYPE"
+FROM popolo_ownership
+WHERE owner_organization_id is null
+;
+\COPY (SELECT * FROM pov) TO '/var/lib/postgresql/neo4j_import/rels_personal_ownerships.csv' WITH CSV HEADER
+
+
+-- organization_ownerships.csv
+CREATE TEMP VIEW oov as
+SELECT
+  md5(owner_organization_id::text || ':' || owned_organization_id::text || ':' || coalesce(start_date, 'NULL'))::uuid as "id",
+  owner_organization_id as ":START_ID(Organization-ID)",
+  owned_organization_id as ":END_ID(Organization-ID)",
+  percentage,
+  start_date as "start_date:date",
+  date_2_ts(start_date) as "start_date_ts:long",
+  end_date as "end_date:date",
+  date_2_ts(end_date) as "end_date_ts:long",
+  end_reason,
+  CASE WHEN end_date > current_date::text or end_date is null THEN 'true'
+       ELSE 'false'
+  END as "is_current:boolean",
+  'DETIENE_QUOTA_DI'::text as ":TYPE"
+FROM popolo_ownership
+WHERE owner_person_id is null
+;
+\COPY (SELECT * FROM oov) TO '/var/lib/postgresql/neo4j_import/rels_organization_ownerships.csv' WITH CSV HEADER
+
+-- appointments.csv
+CREATE TEMP VIEW app as
+SELECT
+  md5(am.person_id::text || ':' || pm.person_id::text || ':' || pm.label || ':' || coalesce(pm.start_date, 'NULL')::text)::uuid as "id",
+  am.person_id as ":START_ID(Person-ID)",
+  pm.person_id as ":END_ID(Person-ID)",
+  pm.role,
+  pm.label,
+  pm.start_date as "start_date:date",
+  date_2_ts(pm.start_date) as "start_date_ts:long",
+  pm.end_date as "end_date:date",
+  date_2_ts(pm.end_date) as "end_date_ts:long",
+  pm.end_reason,
+  CASE WHEN pm.end_date > current_date::text or pm.end_date is null THEN 'true'
+       ELSE 'false'
+  END as "is_current:boolean",
+  'HA_NOMINATO'::text as ":TYPE"
+from popolo_membership pm, popolo_membership am
+where pm.appointed_by_id is not null and pm.appointed_by_id=am.id
+;
+\COPY (SELECT * FROM app) TO '/var/lib/postgresql/neo4j_import/rels_appointments.csv' WITH CSV HEADER

compose/update_neo4j_data.sh

+if [[ -f /root/.linkurious_env ]]; then
+    source /root/.linkurious_env
+    echo LINKURIOUS env variables read
+else
+    echo "No variables set. ERROR."
+    exit 1
+fi
+
+if [ -z ${NEO4J_USER} ]; then
+  echo "NEO4J_USER must be set. ERROR."
+  exit 1
+fi
+
+if [[ -z ${NEO4J_PASSWORD} ]]; then
+    echo "NEO4J_PASSWORD must be set. ERROR."
+    exit 1
+fi
+
+if [[ -z ${LINKURIOUS_USER} ]]; then
+    echo "LINKURIOUS_USER must be set. ERROR."
+    exit 1
+fi
+
+if [[ -z ${LINKURIOUS_PASSWORD} ]]; then
+    echo "LINKURIOUS_PASSWORD must be set. ERROR."
+    exit 1
+fi
+
+if [[ -z ${LINKURIOUS_SOURCE_KEY} ]]; then
+    echo "LINKURIOUS_SOURCE_KEY must be set. ERROR."
+    exit 1
+fi
+
+if [[ -z ${LINKURIOUS_REINDEX} ]]; then
+  echo "Setting LINKURIOUS_REINDEX to true."
+  export LINKURIOUS_REINDEX=true
+fi
+
+if [[ -z ${NEO4J_DOCKER_VOLUME_PATH} ]]; then
+  echo "Setting NEO4J_DOCKER_VOLUME_PATH to default /var/lib/... "
+  export NEO4J_DOCKER_VOLUME_PATH=/var/lib/docker/volumes/opdmservice_neo4j_data/_data/databases
+fi
+
+echo " "
+echo " ---------------------------------------"
+echo "| Starting update procedure using:      "
+echo "| using:                                "
+echo "| LK_USER: ${LINKURIOUS_USER}           "
+echo "| LK_PASSWORD: ${LINKURIOUS_PASSWORD}   "
+echo "| N4J_USER: ${NEO4J_USER}               "
+echo "| N4J_PASSWORD: ${NEO4J_PASSWORD}       "
+echo "| SOURCE_KEY: ${LINKURIOUS_SOURCE_KEY}  "
+echo "| REINDEX: ${LINKURIOUS_REINDEX}        "
+echo "| NEO4J_DOCKER_VOLUME_PATH: ${NEO4J_DOCKER_VOLUME_PATH} "
+echo " ---------------------------------------"
+
+# csv_extraction, csv_split & csv_transfer
+echo " "
+echo " -----------------------------"
+echo "| Fetching data from postgres |"
+echo " -----------------------------"
+docker exec -t opdm-service_postgres bash -c "cd /var/lib/postgresql/neo4j_import && ./extract_neo4j_csv.sh"
+
+# neo4j_import_data
+echo " "
+echo " -------------------------------------"
+echo "| Importing data into new neo4j graph |"
+echo " -------------------------------------"
+docker exec -t -e HEAP_SIZE=2G \
+  opdm-service_neo4j neo4j-admin import --database=graph_new.db --multiline-fields=true \
+    --nodes="import/nodes_areas_header.csv,import/nodes_areas.csv" \
+    --nodes="import/nodes_persons_header.csv,import/nodes_persons.csv" \
+    --nodes="import/nodes_organizations_header.csv,import/nodes_organizations.csv" \
+    --relationships="import/rels_memberships_header.csv,import/rels_memberships.csv" \
+    --relationships="import/rels_areas_parents_header.csv,import/rels_areas_parents.csv" \
+    --relationships="import/rels_organizations_parents_header.csv,import/rels_organizations_parents.csv" \
+    --relationships="import/rels_organizations_areas_parents_header.csv,import/rels_organizations_areas_parents.csv" \
+    --relationships="import/rels_personal_ownerships_header.csv,import/rels_personal_ownerships.csv" \
+    --relationships="import/rels_organization_ownerships_header.csv,import/rels_organization_ownerships.csv" \
+    --relationships="import/rels_appointments_header.csv,import/rels_appointments.csv"
+
+# neo4j_swap_graph_data
+pushd $NEO4J_DOCKER_VOLUME_PATH || exit
+rm -rf graph_old.db
+chown -R 101:101 graph_new.db
+mv graph.db graph_old.db
+mv graph_new.db graph.db
+popd || exit
+
+# neo4j_restart
+echo " "
+echo " --------------------------------------"
+echo "| Restarting neo4j ...                 |"
+echo " --------------------------------------"
+docker restart opdm-service_neo4j
+
+
+if [ $LINKURIOUS_REINDEX == true ]; then
+
+  # linkurious_reconnect
+  curl -s -c cookies -X POST -d "usernameOrEmail=${LINKURIOUS_USER}" -d "password=${LINKURIOUS_PASSWORD}" "http://linkurious.openpolis.io/api/auth/login" > /dev/null
+  echo " "
+  echo " -------------------------------------------"
+  echo "| Reconnecting linkurious to the datasource |"
+  echo " -------------------------------------------"
+  export CONFIG_INDEX=$(curl -s -b cookies "http://linkurious.openpolis.io/api/admin/sources" | jq ".[] | select(.key == \"${LINKURIOUS_SOURCE_KEY}\") | .configIndex")
+  curl -s -X POST -b cookies "http://linkurious.openpolis.io/api/admin/source/${CONFIG_INDEX}/connect"
+  echo "Waiting for connection ..."
+  status=$(curl -s -b cookies "http://linkurious.openpolis.io/api/admin/sources" | jq ".[] | select(.key == \"${LINKURIOUS_SOURCE_KEY}\") | .state")
+  until [ $status == "\"ready\"" ] || [ $status == "\"needReindex\"" ]
+  do
+      echo $status
+      sleep 5;
+      status=$(curl -s -b cookies "http://linkurious.openpolis.io/api/admin/sources" | jq ".[] | select(.key == \"${LINKURIOUS_SOURCE_KEY}\") | .state")
+  done
+  echo "... reconnected!"
+
+  # rebuild neo4j alternative indexes
+  docker exec -it opdm-service_neo4j cypher-shell --username ${NEO4J_USER} --password ${NEO4J_PASSWORD} \
+	  "call db.index.fulltext.createNodeIndex('myAlternativeNodeIdIndex', ['Area', 'Organizzazione', 'Persona'], ['id'], {analyzer: 'keyword'})"
+  echo "Generating myAlternativeNodeIdIndex"
+  sleep 5
+
+  docker exec -it opdm-service_neo4j cypher-shell --username ${NEO4J_USER} --password ${NEO4J_PASSWORD} \
+	  "call db.index.fulltext.createRelationshipIndex('myAlternativeEdgeIdIndex', ['ATTIVA_IN', 'DETIENE_QUOTA_DI', 'HA_NOMINATO', 'INCARICO_IN', 'PARTE_DI'], ['id'], {analyzer: 'keyword'})"
+  echo "Generating myAlternativeEdgeIdIndex"
+  sleep 5
+
+  # linkurious_reindex
+  echo " "
+  echo " ----------------------------"
+  echo "| Reindexing linkurious data |"
+  echo " ----------------------------"
+  curl -s -X POST -b cookies "http://linkurious.openpolis.io/api/${LINKURIOUS_SOURCE_KEY}/search/index"
+  while [ $(curl -s -b cookies "http://linkurious.openpolis.io/api/${LINKURIOUS_SOURCE_KEY}/search/status" | jq .indexing) == "\"ongoing\"" ]
+  do
+      curl -s -b cookies "http://linkurious.openpolis.io/api/${LINKURIOUS_SOURCE_KEY}/search/status" | jq '"\(.indexing_status) - \(.indexing_progress)"'
+      sleep 30;
+  done
+  rm cookies
+fi
+
+echo " "
+echo " ------------"
+echo "|   Ready!   |"
+echo " ------------"

config/.gitignore

-/.dcenv
+/.dcenv*
 /.env

config/settings/__init__.py

+from .base import *  # noqa

config/settings/test.py

+# noinspection PyUnresolvedReferences
+from .base import *  # noqa
+
+# https://docs.djangoproject.com/en/dev/ref/settings/#test-runner
+TEST_RUNNER = "django.test.runner.DiscoverRunner"
+
+# CACHES
+# ------------------------------------------------------------------------------
+# https://docs.djangoproject.com/en/dev/ref/settings/#caches
+CACHES = {
+    "default": {
+        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+        "LOCATION": "",
+    }
+}
+
+# PASSWORDS
+# ------------------------------------------------------------------------------
+# https://docs.djangoproject.com/en/dev/ref/settings/#password-hashers
+PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
+
+# EMAIL
+# ------------------------------------------------------------------------------
+# https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
+EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
+
+# Audit log settings
+# -----------------------------------------------------------------------------
+# Turn off audit log during tests
+DJANGO_EASY_AUDIT_WATCH_MODEL_EVENTS = False
+DJANGO_EASY_AUDIT_WATCH_AUTH_EVENTS = False
+DJANGO_EASY_AUDIT_WATCH_REQUEST_EVENTS = True
+
+# HAYSTACK_CONNECTIONS = {
+#     "default": {"ENGINE": "haystack.backends.simple_backend.SimpleEngine"}
+# }

config/urls.py

@@ -6,7 +6,11 @@ from django.views.generic import TemplateView, RedirectView
 from drf_yasg import openapi
 from drf_yasg.views import get_schema_view
 from rest_framework.documentation import include_docs_urls
-from rest_framework_jwt.views import obtain_jwt_token, refresh_jwt_token
+from rest_framework_simplejwt.views import (
+    TokenObtainSlidingView,
+    TokenRefreshSlidingView,
+    TokenVerifyView,
+)
 
 from project.views import AboutView, DatalogView, ChangelogView
 
@@ -21,26 +25,59 @@ schema_view = get_schema_view(api_info, validators=["flex"], public=True)
 urlpatterns = [
     url(r"^$", RedirectView.as_view(url="about"), name="home"),
     url(r"^about/$", AboutView.as_view(template_name="about.html"), name="about"),
-    url(r"^about/it/$", AboutView.as_view(template_name="about_it.html"), name="about_it"),
-    url(r"^data-changelog/$", DatalogView.as_view(template_name="changelog.html"), name="datalog"),
-    url(r"^changelog/$", ChangelogView.as_view(template_name="changelog.html"), name="changelog"),
+    url(
+        r"^about/it/$",
+        AboutView.as_view(template_name="about_it.html"),
+        name="about_it",
+    ),
+    url(
+        r"^data-changelog/$",
+        DatalogView.as_view(template_name="changelog.html"),
+        name="datalog",
+    ),
+    url(
+        r"^changelog/$",
+        ChangelogView.as_view(template_name="changelog.html"),
+        name="changelog",
+    ),
     url(r"^docs/", include_docs_urls(title=api_title), name="docs"),
     url(r"^admin/", admin.site.urls),
-    url(r"^schema/", schema_view.with_ui("swagger", cache_timeout=None), name="schema-swagger"),
-    url(r"^redoc/", schema_view.with_ui("redoc", cache_timeout=None), name="schema-redoc"),
+    url(
+        r"^schema/",
+        schema_view.with_ui("swagger", cache_timeout=None),
+        name="schema-swagger",
+    ),
+    url(
+        r"^redoc/",
+        schema_view.with_ui("redoc", cache_timeout=None),
+        name="schema-redoc",
+    ),
     url(r"^403$", TemplateView.as_view(template_name="403.html"), name="tampering-403"),
     # External apps
     url(r"^popolo/", include("popolo.urls")),
     url(r"^taskmanager/", include("taskmanager.urls")),
-    url(r"^admin/uwsgi/", include("django_uwsgi.urls")),
+    # url(r"^admin/uwsgi/", include("django_uwsgi.urls")),
     # REST API authentication endpoints
-    url(r"^api-token-auth/", obtain_jwt_token, name="obtain-jwt"),
-    url(r"^api-token-refresh/", refresh_jwt_token, name="refresh-jwt"),
+    url(
+        r"^api-token-auth/",
+        TokenObtainSlidingView.as_view(),
+        name="simplejwt_obtain_token",
+    ),
+    url(
+        r"^api-token-refresh/",
+        TokenRefreshSlidingView.as_view(),
+        name="simplejwt_refresh_token",
+    ),
+    url(
+        r"^api-token-verify/", TokenVerifyView.as_view(), name="simplejwt_verify_token"
+    ),
     url(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")),
     # REST API v1
     url(r"^(?P<version>v1)/", include("project.api_v1.urls")),
+    url(r"^(?P<version>v1)/", include("project.api_v1.urls")),
 ]
 
 if settings.DEBUG_TOOLBAR:
     import debug_toolbar
+
     urlpatterns.append(url(r"^__debug__/", include(debug_toolbar.urls)))

docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.5"
 services:
 
   web:
-    container_name: opdm-service_web_1
+    container_name: opdm-service_web
     restart: always
     image: openpolis/opdm/opdm-service:latest
     expose:
@@ -15,7 +15,7 @@ services:
     environment:
       - DATABASE_URL=postgis://${POSTGRES_USER}:${POSTGRES_PASS}@postgres/${POSTGRES_DB}
       - DEBUG=${DEBUG}
-      - HAYSTACK_SIGNAL_PROCESSOR=project.api_v1.core.PersonOnlySignalProcessor
+      - HAYSTACK_SIGNAL_PROCESSOR=project.core.PersonOnlySignalProcessor
       - DEBUG_TOOLBAR_INTERNAL_IPS=172.20.0.*
       - DJANGO_CORS_HEADERS=on
       - CORS_ORIGIN_WHITELIST=${CORS_ORIGIN_WHITELIST}
@@ -27,23 +27,23 @@ services:
       - MEDIA_ROOT=/app/public/media
       - STATIC_ROOT=/app/public/static
       - DEFAULT_FROM_EMAIL=${DEFAULT_FROM_EMAIL}
-      - SERVER_EMAIL=${SERVER_EMAIL}
       - ADMIN_EMAIL=${ADMIN_EMAIL}
       - SOLR_URL=${SOLR_URL}
       - ATOKA_API_ENDPOINT=${ATOKA_API_ENDPOINT}
       - ATOKA_API_VERSION=${ATOKA_API_VERSION}
       - ATOKA_API_KEY=${ATOKA_API_KEY}
-      - SLACK_TOKEN=${SLACK_TOKEN}
+      - UWSGI_TASKMANAGER_NOTIFICATIONS_SLACK_TOKEN=${UWSGI_TASKMANAGER_NOTIFICATIONS_SLACK_TOKEN}
+      - UWSGI_TASKMANAGER_NOTIFICATIONS_SLACK_CHANNELS=${UWSGI_TASKMANAGER_NOTIFICATIONS_SLACK_CHANNELS}
       - CI_COMMIT_SHA=${CI_COMMIT_SHA}
     volumes:
       - public:/app/public
       - data:/app/data
       - uwsgi_spooler:/var/lib/uwsgi
       - weblogs:/var/log
-    command: /usr/local/bin/uwsgi --socket=:8000 --master --logto=/var/log/uwsgi.log --env DJANGO_SETTINGS_MODULE=config.settings --pythonpath=/app --module=wsgi --callable=application --processes=4 --spooler=/var/lib/uwsgi --spooler-processes=2
+    command: /usr/local/bin/uwsgi --socket=:8000 --master --env DJANGO_SETTINGS_MODULE=config.settings --pythonpath=/app --module=config.wsgi --callable=application --processes=4 --spooler=/var/lib/uwsgi --spooler-processes=2
 
   nginx:
-    container_name: opdm-service_nginx_1
+    container_name: opdm-service_nginx
     restart: always
     build: compose/nginx/
     expose:
@@ -58,11 +58,12 @@ services:
       - web:web
 
   postgres:
-    container_name: opdm-service_postgres_1
+    container_name: opdm-service_postgres
     restart: always
     image: mdillon/postgis:9.6-alpine
     volumes:
       - pg_data:/var/lib/postgresql/data
+      - neo4j_import:/var/lib/postgresql/neo4j_import
     environment:
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_USER=${POSTGRES_USER}
@@ -71,15 +72,17 @@ services:
       - "5432"
 
   redis:
-    container_name: opdm-service_redis_1
+    container_name: opdm-service_redis
     restart: always
     image: redis:latest
+    volumes:
+      - redis_data:/data
     expose:
       - "6379"
 
   solr:
     restart: always
-    container_name: opdm-service_solr_1
+    container_name: opdm-service_solr
     build: compose/solr/
     ports:
      - "8983"
@@ -91,6 +94,27 @@ services:
       - opdm
       - /opt/solr/server/solr/mycores/opdm
 
+  neo4j:
+    restart: always
+    container_name: opdm-service_neo4j
+    image: neo4j:3.5.12-enterprise
+    ports:
+      - "7474:7474"
+      - "7687:7687"
+    environment:
+      - NEO4J_dbms_security_procedures_unrestricted=apoc.*
+      - NEO4J_apoc_import_file_enabled=true
+      - NEO4J_apoc_export_file_enabled=true
+      - NEO4J_AUTH=neo4j/${NEO4J_PASS}
+      - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes
+      - SECURE_FILE_PERMISSIONS=yes
+    volumes:
+      - neo4j_conf:/var/lib/neo4j/conf
+      - neo4j_data:/data
+      - neo4j_logs:/logs
+      - neo4j_import:/var/lib/neo4j/import
+      - neo4j_plugins:/var/lib/neo4j/plugins
+
 volumes:
   public:
     name: opdmservice_public
@@ -98,12 +122,24 @@ volumes:
     name: opdmservice_data
   pg_data:
     name: opdmservice_pg_data
+  neo4j_logs:
+    name: opdmservice_neo4j_logs
+  neo4j_data:
+    name: opdmservice_neo4j_data
+  neo4j_conf:
+    name: opdmservice_neo4j_conf
+  neo4j_import:
+    name: opdmservice_neo4j_import
+  neo4j_plugins:
+    name: opdmservice_neo4j_plugins
   uwsgi_spooler:
     name: opdmservice_uwsgi_spooler
   weblogs:
     name: opdmservice_weblogs
   solr_data:
     name: opdmservice_solr_data
+  redis_data:
+    name: opdmservice_redis_data
 
 networks:
   default: