CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [1.1.4]
+
+### Changed
+- add pub_part_percentage to OrganizationEconomics 
+- min and max validators corrected for revenue_trend
+- `--shares-level` argument added to import_atoka_extract_json, defaults to zero
+- a limit greater than 50 may be specified in the `get_items_from_ids` atoka connections method
+- `AtokaOwnershipsExtractor` fetch information about owned share also from non-active companies
+
 ## [1.1.3]
 
 ### Fixed

DATA_CHANGELOG.md

@@ -6,10 +6,14 @@ The format is lousely based on [Keep a Changelog](http://keepachangelog.com/en/1
 
 ## [Unreleased]
 
-- public organization's ownerships (and owned organizations) from ATOKA 
 - administrative history of an organ from MININT:
   - memberships' end date and reason
   - electoral lists
+
+## [2019-02-12]
+
+- public organization's ownerships (and owned organizations) from ATOKA
+- memberships into organizations imported from ATOKA
    
 ## [2019-02-08]
 

project/__init__.py

@@ -3,4 +3,4 @@
 Openpolis Data Manager service package (backend)
 """
 
-__version__ = '1.1.3'
+__version__ = '1.1.4'

project/api_v1/management/commands/script_merge_duplicate_organizations_giunte_consigli.py

@@ -34,7 +34,7 @@ class Command(LoggingBaseCommand):
         getattr(self, 'handle_{0}'.format(context))()
         self.logger.info("End")
 
-    def handle_comuni(self):
+    def handle_com(self):
 
         parameters_list = [
             {'ok_name': 'Comune di Brescia', 'ko_name': 'Associazione comuni bresciani',
@@ -102,7 +102,7 @@ class Command(LoggingBaseCommand):
 
         # assign region as parent to consiglio currently missing them
         for reg in ['friuli', 'trentino', 'aosta']:
-            self.logger.info("Assigning regiona s parent to consiglio of {0}".format(reg))
+            self.logger.info("Assigning region's parent to consiglio of {0}".format(reg))
             r = Organization.objects.get(name__icontains=reg, classification="Regione")
             c = Organization.objects.get(name__icontains=reg, classification="Consiglio regionale")
             c.parent = r

project/atoka/connections.py

@@ -99,7 +99,7 @@ class AtokaConn(object):
         return result['items'][0]
 
     def get_items_from_ids(
-        self, ids: list, item_type: str, ids_field_name: str = 'ids', batch_size: int = 50, **kwargs
+        self, ids: list, item_type: str, ids_field_name: str = 'ids', batch_size: int = 50, limit=50, **kwargs
     ) -> list:
         """Transform a request for a list of ids larger than batch_size,
          to a batch request of enough rows with a limit of batch_size, so that all results
@@ -111,6 +111,7 @@ class AtokaConn(object):
         :param item_type: str
         :param ids_field_name: ids, tax_ids
         :param batch_size: size of the number of ids searched by row of the batch IO
+        :param limit: max n. of results for eache batch request
         :param kwargs: - more atoka parameters for filtering results
             (ex: packages=base,shares, active='true', ccia='*')
         :return: results as a list of dicts
@@ -141,10 +142,12 @@ class AtokaConn(object):
         # build fileIO to upload form batch execution
         file_io = StringIO()
         for n, r in enumerate(chunks(ids, batch_size)):
-            print(json.dumps({
-                "reqId": "r{0:05d}".format(n),
-                ids_field_name: ','.join(r),
-            }), file=file_io)
+            for offset in range(0, limit, 50):
+                print(json.dumps({
+                    "reqId": "r{0:05d}".format(int(n*10e4 + offset)),
+                    ids_field_name: ','.join(r),
+                    'offset': offset
+                }), file=file_io)
 
         # batch API request
         fields = {

project/atoka/etl/extractors.py

@@ -92,9 +92,11 @@ class AtokaOwnershipsExtractor(Extractor):
         # fetch all companies among the list having govType values set
         # will need batch_size=1 here, because shares may contain many results
         # and the limit is 50
+        # fetchin companies with all active status, since some of the non-active seems to hold
+        # important ownership information on active ones
         try:
             res_tot = atoka_conn.get_companies_from_tax_ids(
-                tax_ids, packages='base,shares', active='true', batch_size=1
+                tax_ids, packages='base,shares', active='*', batch_size=1
             )
             atoka_companies_requests += len(res_tot)
 
@@ -136,31 +138,35 @@ class AtokaOwnershipsExtractor(Extractor):
             )
         )
 
-        # transform the results into a dict
-        res_dict = {
-            r['base']['taxId']: {
-                'atoka_id': r['id'],
-                'other_atoka_ids': [
+        # transform the results into a dict,
+        # merging results from multiple records corresponding to the same tax_id
+        res_dict = {}
+
+        for r in res_tot:
+            r_dict = {}
+            r_dict.setdefault('atoka_id', r['id'])
+            r_dict.setdefault('other_atoka_ids', [
                     atoka_id for atoka_id in res_doubles[r['base']['taxId']] if atoka_id != r['id']
-                ] if r['base']['taxId'] in res_doubles else [],
-                'name': r['name'],
-                'legal_form': [x['name'] for x in r['base']['legalForms'] if x['level'] == 2][0],
-                'rea': r['base'].get('rea', None),
-                'cciaa': r['base'].get('cciaa', None),
-                'shares_owned': [
-                    {
-                        'name': sho['name'],
-                        'last_updated': sho['lastUpdate'],
-                        'atoka_id': sho['id'],
-                        'percentage': sho['ratio'] * 100.
-                    }
-                    for sho in filter(
-                        lambda x: x['active'] is True and x['typeOfRight'] == 'proprietà' and 'ratio' in x,
-                        r['shares']['sharesOwned']
-                    )
-                ]
-            } for r in res_tot
-        }
+                ] if r['base']['taxId'] in res_doubles else []
+            )
+            r_dict.setdefault('name', r['name'])
+            r_dict.setdefault('legal_form', [x['name'] for x in r['base']['legalForms'] if x['level'] == 2][0])
+            r_dict.setdefault('rea', r['base'].get('rea', None))
+            r_dict.setdefault('cciaa', r['base'].get('cciaa', None))
+            r_dict.setdefault('shares_owned', [])
+            r_dict['shares_owned'].extend(
+                {
+                    'name': sho['name'],
+                    'last_updated': sho['lastUpdate'],
+                    'atoka_id': sho['id'],
+                    'percentage': sho['ratio'] * 100.
+                }
+                for sho in filter(
+                    lambda x: x['active'] is True and x['typeOfRight'] == 'proprietà' and 'ratio' in x,
+                    r['shares']['sharesOwned']
+                )
+            )
+            res_dict[r['base']['taxId']] = r_dict
 
         # extract all atoka_ids from shares_owned elements and returns flat list
         # then apply list(set(x)) to remove duplicates, if any
@@ -170,7 +176,7 @@ class AtokaOwnershipsExtractor(Extractor):
         ]))))
 
         owned_orgs = atoka_conn.get_companies_from_atoka_ids(
-            owned_atoka_ids, packages='base', active='true', batch_size=1
+            owned_atoka_ids, packages='base', active='true', batch_size=10
         )
         atoka_companies_requests += len(owned_orgs)
         self.logger.debug("- ricevuti dettagli per {0} partecipate".format(len(owned_orgs)))
@@ -189,7 +195,7 @@ class AtokaOwnershipsExtractor(Extractor):
 
         # extract all people's atoka_ids from res_owned elements and returns flat list, removing duplicates
         people = atoka_conn.get_roles_from_atoka_ids(
-            owned_atoka_ids, packages='base,companies',
+            owned_atoka_ids, packages='base,companies', limit=150,
             companiesRolesOfficial='true', companiesRoles=atoka_conn.allowed_roles
         )
         atoka_people_requests += len(people)

project/atoka/management/commands/import_atoka_extract_json.py

@@ -37,6 +37,13 @@ class Command(LoggingBaseCommand):
             default="./resources/data/atoka/atoka.json",
             help="Complete path to json file"
         )
+        parser.add_argument(
+            "--shares-level",
+            dest="shares_level",
+            type=int,
+            default=0,
+            help="Level of the public share, starting from public institution = 0"
+        )
 
     def handle(self, *args, **options):
         self.setup_logger(__name__, formatter_key='simple', **options)
@@ -45,19 +52,19 @@ class Command(LoggingBaseCommand):
         offset = options['offset']
         jsonfile = options['jsonfile']
         classifications = options.get('classifications', [])
+        shares_level = options['shares_level']
 
         self.logger.info("Start procedure")
 
+        if shares_level < 0 or shares_level > 2:
+            raise Exception("--shares-level must be between 0 and 2")
+
         # start filtering current organizations with a tax_id,
         # excluding those classified as private
         organizations_qs = Organization.objects.filter(
-            classifications__classification__scheme='FORMA_GIURIDICA_OP'
-        ).current().exclude(
-            classifications__classification_id__in=[
-                11, 20, 24, 29, 48, 69, 83, 295, 321, 346, 403, 621, 941, 730, 1182, 1183, 1184, 1185, 1186, 1187,
-                1188, 1190, 1189, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202
-            ]
-        ).filter(identifier__isnull=False)
+            classifications__classification__scheme='LIVELLO_PARTECIPAZIONE_OP',
+            classifications__classification__code=str(shares_level)
+        ).current().filter(identifier__isnull=False)
 
         if classifications:
             organizations_qs = organizations_qs.filter(classifications__classification_id__in=classifications)

project/atoka/migrations/0002_auto_20190214_1250.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.15 on 2019-02-14 11:50
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('atoka', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='organizationeconomics',
+            name='pub_part_percentage',
+            field=models.PositiveIntegerField(blank=True, help_text='Percentage of shares held or referrable to public institutions', null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(100)]),
+        ),
+        migrations.AlterField(
+            model_name='organizationeconomics',
+            name='revenue_trend',
+            field=models.FloatField(blank=True, help_text='Latest trend in revenues', null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(100.0)]),
+        ),
+        migrations.AlterField(
+            model_name='organizationeconomicshistorical',
+            name='revenue_trend',
+            field=models.FloatField(blank=True, help_text="Current year's revenue trend with respect to previous year", null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(100.0)]),
+        ),
+        migrations.AlterField(
+            model_name='organizationeconomicshistorical',
+            name='year',
+            field=models.PositiveIntegerField(help_text='Year of validity of economics indicator', validators=[django.core.validators.MinValueValidator(1980), django.core.validators.MaxValueValidator(2019)]),
+        ),
+    ]

project/atoka/models.py

@@ -30,7 +30,7 @@ class OrganizationEconomics(models.Model):
     )
     revenue_trend = models.FloatField(
         blank=True, null=True,
-        validators=[MinValueValidator(0.), MaxValueValidator(current_year)],
+        validators=[MinValueValidator(0.), MaxValueValidator(100.)],
         help_text=_("Latest trend in revenues")
     )
     capital_stock = models.BigIntegerField(
@@ -85,6 +85,11 @@ class OrganizationEconomics(models.Model):
         default=False,
         help_text=_("If the organization is listed in the publick stock exchange")
     )
+    pub_part_percentage = models.PositiveIntegerField(
+        blank=True, null=True,
+        validators=[MinValueValidator(0), MaxValueValidator(100)],
+        help_text=_("Percentage of shares held or referrable to public institutions")
+    )
 
 
 class OrganizationEconomicsHistorical(models.Model):
@@ -99,7 +104,7 @@ class OrganizationEconomicsHistorical(models.Model):
         related_name='historical_values'
     )
     year = models.PositiveIntegerField(
-        validators=[MinValueValidator(2000), MaxValueValidator(current_year)],
+        validators=[MinValueValidator(1980), MaxValueValidator(current_year)],
         help_text=_("Year of validity of economics indicator")
     )
     employees = models.PositiveIntegerField(
@@ -112,7 +117,7 @@ class OrganizationEconomicsHistorical(models.Model):
     )
     revenue_trend = models.FloatField(
         blank=True, null=True,
-        validators=[MinValueValidator(0.), MaxValueValidator(current_year)],
+        validators=[MinValueValidator(0.), MaxValueValidator(100.)],
         help_text=_("Current year's revenue trend with respect to previous year")
     )
     capital_stock = models.BigIntegerField(

project/atoka/tests/etl/atoka_mocks.py

@@ -576,7 +576,7 @@ def get_people_atoka_ids_batch(atoka_ids, packages='base,shares', batch_size=1):
 def get_roles_atoka_ids_batch(
     atoka_ids, packages='base,companies',
-    companiesRolesOfficial=None, companiesRoles=None, batch_size=1
+    companiesRolesOfficial=None, companiesRoles=None, batch_size=1, limit=100
 ):
     """Return mocks of lists of people, with roles in companies within atoka_ids

setup.cfg

 [bumpversion]
-current_version = 1.1.3
+current_version = 1.1.4
 commit = True
 tag = True
 tag_name = v{new_version}