.gitignore

-# Basics
+# Created by .ignore support plugin (hsz.mobi)
+Thumbs.db
+ehthumbs.db
+ehthumbs_vista.db
+*.stackdump
+[Dd]esktop.ini
+$RECYCLE.BIN/
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+*.lnk
+.idea/
+cmake-build-debug/
+cmake-build-release/
+*.iws
+out/
+.idea_modules/
+atlassian-ide-plugin.xml
+.idea/replstate.xml
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+.idea/httpRequests
+.DS_Store
+.AppleDouble
+.LSOverride
+Icon
+._*
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-v][a-z]
+[._]sw[a-p]
+Session.vim
+.netrwhist
+*~
+tags
+[._]*.un~
+__pycache__/
 *.py[cod]
-*.pyc
-__pycache__
-
-# Env file
-env
-*.env
-
-# Logs
-*.log
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+*.manifest
+*.spec
 pip-log.txt
-
-# Unit test / coverage reports
+pip-delete-this-directory.txt
+htmlcov/
+.tox/
 .coverage
-.tox
+.coverage.*
+.cache
 nosetests.xml
-
-# Translations
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
 *.mo
 *.pot
-
-# Pycharm
-.idea
-
-# Vim
-
-*~
-*.swp
-*.swo
-
-# npm
-node_modules/
-
-# Campass
-.sass-cache
-
-# osx .DS_Store
-.DS_Store
-
-.gitattributes
-
-/venv
-dump*
-
+*.log
+local_settings.py
+db.sqlite3
+instance/
+.webassets-cache
+.scrapy
+docs/_build/
+docs/_static/
+target/
 .ipynb_checkpoints
 .python-version
+celerybeat-schedule
+*.sage.py
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.spyderproject
+.spyproject
+.ropeproject
+/site
+.mypy_cache/
+.fuse_hidden*
+.directory
+.Trash-*
+.nfs*
+
+*.rdb
 
-*.zip
-*.egg 
 /data
 resources/data/bdap_anagrafe_enti.csv
 resources/data/parsers

CHANGELOG.md

 # Changelog
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
@@ -6,15 +7,40 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+## [1.1.15]
+
+### Changed
+- import_orgs_from_json refactored, to use JsonDiff class;
+- import_ownerships_from_json refactored, to use JsonDiff class;
+  
+## [1.1.14]
+
 ### Added
-- `v1/memberships` now accepts PATCH request, enabling the (partial) update of a whole batch of `Membership objects.
-- Add [`Interval` validator](./project/api_v1/validators.py) to be used when serializing models with `start_date` and 
-  `end_date` (`Dateframeable)
+- ETL procedures from ATOKA refined.
+- It is now possible to import json data differentially.
+- JsonDiffCloseItemsInRemovalsCompositeETL allows differential import
+  of items that are closed when missing elements are detected in diffs.
+
+### Changed
+- The script to merge organizations now merge the memberships, before 
+  removing 'old' organizations.   
 
 ## [1.1.13]
 
+### Added
+
+- Script to transfer person's headshots to S3 implemented.
+- Import of information from [atoka], starting from persons in OPDM is now possible.
+- Person and Organization appear now in the admin site.
+- Thresholds for AKAs can now be defined dynamically, as management task's parameters.
+- `v1/memberships` now accepts PATCH request, enabling the (partial) update of a whole batch of 
+  `Membership objects.
+- Add [`Interval` validator](./project/api_v1/validators.py) to be used when serializing models with
+  `start_date` and `end_date` (`Dateframeable).
+
 ### Changed
-- pool of level-0 organizations restricted: MIUR orgs removed
+
+- Pool of level-0 organizations restricted: MIUR organizations removed.
 
 ## [1.1.12]
 
@@ -30,13 +56,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   this avoids missing organizations log messages;
 
 ### Added
-- fullAddress property from atoka imported into all organizations, as
-  contact_detail of type MAIL
+- `fullAddress` property from [atoka] imported into all Organizations, as contact_detail of type MAIL.
 
 ## [1.1.10]
 
 ### Added
-- shareholders importes along with sharesOwned from ATOKA
+- Import shareholders along with sharesOwned from [atoka].
 
 ## [1.1.9]
 
@@ -44,45 +69,47 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - script that computes participation level starts with broader criterion,
   setting all organizations having classification TIPOLOGIA_IPA_BDAP set to PA
-- atoka extractor considers additional CFs in identifiers with scheme 
+- [atoka] extractor considers additional CFs in identifiers with scheme 
   ALTRI_CF_ATOKA, when creating the batches of tax_ids to fetch
-- atoka extractor extracts shares even when percentage is null;
+- [atoka] extractor extracts shares even when percentage is null;
 - organizations lookup in opdm core OrganizationUtils looks for identifiers 
   within ALTRI_CF_ATOKA identifiers
 
 ## [1.1.8]
 
 ### Added
-- ATECO classifications for organizations are imported from ATOKA
+
+- ATECO classifications for organizations are imported from [[atoka]]
 
 ## [1.1.7]
 
 ### Added
 
 - script to compute public interest and classify participation level
-- meta script to launch all upgrade tasks from atoka implemented
+- meta script to launch all upgrade tasks from [atoka] implemented
 
 ## [1.1.6]
 
 ### Changed
-- mechanism to fetch all results from atoka, exceeding the maximum limit improved
-
+- mechanism to fetch all results from [atoka], exceeding the maximum limit improved
 
 ## [1.1.5]
 
 ### Fixed
-- bug on merge of same tax_id info in different atoka records, during extraction corrected;
+- bug on merge of same tax_id info in different [atoka] records, during extraction corrected;
 
 ### Added
-- macro management task composing atoka imports implemented
+- macro management task composing [atoka] imports implemented
 
 ## [1.1.4]
 
+### Added
+- `pub_part_percentage` field to `OrganizationEconomics` model.
+- - `--shares-level` argument added to `import_atoka_extract_json` task, defaults to zero
+
 ### Changed
-- add pub_part_percentage to OrganizationEconomics 
 - min and max validators corrected for revenue_trend
-- `--shares-level` argument added to import_atoka_extract_json, defaults to zero
-- a limit greater than 50 may be specified in the `get_items_from_ids` atoka connections method
+- a limit greater than 50 may be specified in the `get_items_from_ids` [atoka] connections method
 - `AtokaOwnershipsExtractor` fetch information about owned share also from non-active companies
 
 ## [1.1.3]
@@ -97,25 +124,25 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   economics indicators: employees, revenue, capital_stock
 
 ### Changed
-- atoka_persons_memberships.json is now composed of unique persons with multiple memberships
+- atoka_persons_memberships.json is now composed of unique `Persons` with multiple `Memberships`
   optimising number of similarities that will be created and avoiding double similarities
 
 ## [1.1.2]
 
 ### Fixed
 - founding_date is imported correctly
-- double FORMA_GIURIDICA_OP classification is avoided when importing from ATOKA
-- RoleTypes generation from atoka mapping corrected
+- double FORMA_GIURIDICA_OP classification is avoided when importing from [atoka]
+- RoleTypes generation from [atoka] mapping corrected
 - Commissario straordinario removed from atoka_roles_skip_list
 
 ### Changed
-- CCIAA classification and REA identifier from ATOKA moved into one CCIAA-REA identifier
+- CCIAA classification and REA identifier from [atoka] moved into one CCIAA-REA identifier
 - It is now impossible to add more than one classifications of the same scheme to an object (popolo).
 
 ## [1.1.1]
 
 ### Added
-- management task to import OrganizationEconomics, with historical records, from ATOKA 
+- management task to import OrganizationEconomics, with historical records, from [atoka] 
 
 ### Changed
 - is_public flag (SPA) and capital_stock added to OrganizationEconomics
@@ -124,23 +151,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
      
 ## [1.1.0]
 
-### Fixed
-- unofficial roles are not imported from ATOKA anymore
-
 ### Added
-- project.atoka module added, to handle all atoka-related logic and tasks
-- OrganizationEconomics model added to atoka module
+- project.[atoka] module added, to handle all [atoka]-related logic and tasks
+- OrganizationEconomics model added to [atoka] module
 
 ### Changed
-- all atoka-related code moved into project.atoka module 
+- Unofficial roles are no longer imported from [atoka].
+- [atoka]-related code moved into project.[atoka] module (which is a Django app).
 
 ## [1.0.3]
 
 ### Changed
 
-- contexts may be specified for atoka transformations 
+- contexts may be specified for [atoka] transformations 
   in import_atoka_transform management task, so that now the
-  complete atoka import procedure described in the gitlab snippet
+  complete [atoka] import procedure described in the gitlab snippet
   (https://gitlab.depp.it/openpolis/opdm/opdm-service/snippets/26), 
   can be launched
 
@@ -150,7 +175,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - `--use-dummy-transformation` added to import_persons_memberships_from_json task
   to keeb back compatibility with national level imports and use it in
-  new atoka imports
+  new [atoka] imports
 - `--log-step` and `--context` parameters added to import_persons_memberships_from_json
   to allow for setting context string in loader_context (similarities)
 
@@ -158,7 +183,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [1.0.1]
 
 ### Added
-- ETL tasks for Atoka integration
+- ETL tasks for [atoka] integration
 - maintenance script for organizations missing classifications
 - maintenance script for duplicate organization merge
 - data changelog and main changelog urls and views added and linked to
@@ -168,7 +193,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [1.0.0]
 
 ### Added
-- import of ownerships from atoka's API
+- import of ownerships from [atoka]'s API
 - /keyevents endpoint added in API, to handel KeyEvents
 - /organizations API endpoint manages the `key_events` relations
 - organizations loader created under `etl/loaders`
@@ -426,6 +451,9 @@ in the test stage.
 ### Added
 - first release: areas and organizations import management tasks added
 
+
+[atoka]: https://atoka.io
+[1.1.13]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.11...v1.1.13
 [1.1.12]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.11...v1.1.12
 [1.1.11]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.10...v1.1.11
 [1.1.10]: https://gitlab.depp.it/openpolis/opdm/opdm-service/compare/v1.1.9...v1.1.10

LICENSE.txt

@@ -2,7 +2,7 @@
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
 
-Copyright (c) 2017, Fondazione Openpolis <http://www.openpolis.it/>
+Copyright (c) 2019, Fondazione Openpolis <http://www.openpolis.it/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 

config/.env.sample

 # Openpolis Data Manager environment variables
-PROJECT_TITLE=Openpolis DataManager Service
-SECRET_KEY=-not-so-secret
+# SECRET_KEY=not-so-secret
 
-# ADMIN_NAME=admin
 # ADMIN_EMAIL=admin@example.com
-# DEFAULT_FROM_EMAIL=$ADMIN_EMAIL
+# ADMIN_NAME=admin
+# DEFAULT_FROM_EMAIL=admin@example.com
 
 DATABASE_URL=postgresql://postgres:@localhost/opdm
-# CACHE_URL=memcache://127.0.0.1:11211
-# REDIS_URL=redis://127.0.0.1:6379:1
 
-# media and static files may be served from totally different directories
-# in production. For example:
-# MEDIA_ROOT=/home/openpolis.io/public/media
-# STATIC_ROOT=/home/openpolis.io/public/static
+# media and static files may be served from totally different directories in production.
+# For example:
+# MEDIA_ROOT=/path/to/media
+# STATIC_ROOT=/path/to/static
+# LOGS_PATH=/path/to/logs
+
+# Caches configuration
+# USE_REDIS_CACHE=off
+# REDIS_URL=redis://127.0.0.1:6379:1
 
 # Development only
-DEBUG=on
-DEBUG_TOOLBAR=on
-DEBUG_TOOLBAR_INTERNAL_IPS=localhost,127.0.0.1,172.20.0.5
+# DEBUG=on
+# DEBUG_TOOLBAR=on
+# DEBUG_TOOLBAR_INTERNAL_IPS=localhost,127.0.0.1,172.20.0.5
+
+# Haystack configuration
+# SOLR_URL=http://localhost:8983/solr/opdm
+# SOLR_BATCH_SIZE=100
+# HAYSTACK_SEARCH_RESULTS_PER_PAGE=50
+
+# CORS configuration
+DJANGO_CORS_HEADERS=False
+# CORS_ORIGIN_WHITELIST=openpolis.it,www.openpolis.it
+# CORS_ALLOW_CREDENTIALS=True
+
+# Atoka API configuration
+# ATOKA_API_ENDPOINT="https://api.atoka.io"
+# ATOKA_API_VERSION="v2"
+# ATOKA_API_KEY="insert-your-atoka-key"
+
+# Slack integration
+# SLACK_TOKEN="insert-your-slack-token"
+# SLACK_ICON_URL="https://example.com/icon.png"
 
 # Production only
 # ALLOWED_HOSTS=localhost,127.0.0.1
@@ -29,7 +50,6 @@ DEBUG_TOOLBAR_INTERNAL_IPS=localhost,127.0.0.1,172.20.0.5
 # EMAIL_SUBJECT_PREFIX="[Openpolis DataManager] "
 # EMAIL_USE_TLS=True
 
-# CORS configuration
-DJANGO_CORS_HEADERS=False
-# CORS_ORIGIN_WHITELIST=openpolis.it,www.openpolis.it
-# CORS_ALLOW_CREDENTIALS=True
+# AWS credentials
+# AWS_ACCESS_KEY_ID=""
+# AWS_SECRET_ACCESS_KEY=""

docker-compose.yml

@@ -21,6 +21,8 @@ services:
       - CORS_ORIGIN_WHITELIST=${CORS_ORIGIN_WHITELIST}
       - DJANGO_LOG_LEVEL=INFO
       - LOGS_PATH=/var/log
+      - IMPORT_CACHE_PATH=${IMPORT_CACHE_PATH}
+      - IMPORT_OUT_PATH=${IMPORT_OUT_PATH}
       - ALLOWED_HOSTS=${ALLOWED_HOSTS}
       - MEDIA_ROOT=/app/public/media
       - STATIC_ROOT=/app/public/static