CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.1.10]
+
+### Add
+- shareholders importe along with sharesOwned from ATOKA
+
 ## [1.1.9]
 
 ### Add

project/__init__.py

@@ -3,4 +3,4 @@
 Openpolis Data Manager service package (backend)
 """
 
-__version__ = '1.1.9'
+__version__ = '1.1.10'

project/atoka/etl/extractors.py

@@ -158,7 +158,8 @@ class AtokaOwnershipsExtractor(Extractor):
                     'legal_form': [x['name'] for x in r['base']['legalForms'] if x['level'] == 2][0],
                     'rea': r['base'].get('rea', None),
                     'cciaa': r['base'].get('cciaa', None),
-                    'shares_owned': []
+                    'shares_owned': [],
+                    'shareholders':  [],
                 }
             else:
                 r_dict = res_dict[r['base']['taxId']]
@@ -175,6 +176,18 @@ class AtokaOwnershipsExtractor(Extractor):
                     r['shares']['sharesOwned']
                 )
             )
+            r_dict['shareholders'].extend(
+                {
+                    'name': sho['name'],
+                    'last_updated': sho['lastUpdate'],
+                    'atoka_id': sho['id'],
+                    'percentage': sho.get('ratio', 0.) * 100.
+                }
+                for sho in filter(
+                    lambda x: x['active'] is True and x['typeOfRight'] == 'proprietà',
+                    r['shares'].get('shareholders', [])
+                )
+            )
             res_dict[r['base']['taxId']] = r_dict
 
         # extract all atoka_ids from shares_owned elements and returns flat list
@@ -187,8 +200,6 @@ class AtokaOwnershipsExtractor(Extractor):
         owned_orgs = atoka_conn.get_companies_from_atoka_ids(
             owned_atoka_ids, packages='base', active='true', batch_size=10
         )
-        atoka_companies_requests += len(owned_orgs)
-        self.logger.debug("- ricevuti dettagli per {0} partecipate".format(len(owned_orgs)))
 
         owned_orgs_dict = {
             r['id']: {
@@ -203,6 +214,33 @@ class AtokaOwnershipsExtractor(Extractor):
             } for r in owned_orgs
         }
 
+        # extract all atoka_ids from shareholders elements and returns flat list
+        # then apply list(set(x)) to remove duplicates, if any
+        owners_atoka_ids = list(set(list(itertools.chain.from_iterable([
+            [x['atoka_id'] for x in r['shareholders']]
+            for r in res_dict.values()
+        ]))))
+
+        owners_orgs = atoka_conn.get_companies_from_atoka_ids(
+            owners_atoka_ids, packages='base', active='true', batch_size=10
+        )
+
+        owner_orgs_dict = {
+            r['id']: {
+                'name': r['name'],
+                'cciaa': r['base'].get('cciaa', None),
+                'rea': r['base'].get('rea', None),
+                'tax_id': r['base'].get('taxId', None),
+                'vat': r['base'].get('vat', None),
+                'founded': r['base'].get('founded', None),
+                'legal_form': [x['name'] for x in r['base']['legalForms'] if x['level'] == 2][0],
+                'ateco': r['base'].get('ateco', [])
+            } for r in owners_orgs
+        }
+
+        atoka_companies_requests += len(owned_orgs) + len(owners_orgs)
+        self.logger.debug("- ricevuti dettagli per {0} partecipate".format(len(owned_orgs) + len(owners_orgs)))
+
         # extract all people's atoka_ids from res_owned elements and returns flat list, removing duplicates
         people = atoka_conn.get_roles_from_atoka_ids(
             owned_atoka_ids, packages='base,companies',
@@ -284,19 +322,29 @@ class AtokaOwnershipsExtractor(Extractor):
                     )
 
         # upgrade res_dict values with details values
-        for tax_id, owner in res_dict.items():
-            for owned in owner['shares_owned']:
+        for tax_id, org in res_dict.items():
+            for owned in org['shares_owned']:
                 owned_details = owned_orgs_dict.get(owned['atoka_id'], None)
                 if owned_details:
                     for f in ['name', 'cciaa', 'rea', 'ateco', 'tax_id', 'vat', 'founded', 'legal_form', 'roles']:
                         if f in owned_details:
                             owned[f] = owned_details[f]
-
                 else:
                     self.logger.warning(
                         "! organizzazione {0} richiesta ad atoka, "
                         "ma non presente nei risultati".format(owned['atoka_id'])
                     )
+            for owner in org['shareholders']:
+                owner_details = owner_orgs_dict.get(owner['atoka_id'], None)
+                if owner_details:
+                    for f in ['name', 'cciaa', 'rea', 'ateco', 'tax_id', 'vat', 'founded', 'legal_form']:
+                        if f in owner_details:
+                            owner[f] = owner_details[f]
+                else:
+                    self.logger.warning(
+                        "! organizzazione {0} richiesta ad atoka, "
+                        "ma non presente nei risultati".format(owner['atoka_id'])
+                    )
 
         # returns a list
         results = []

project/atoka/etl/transformations.py

@@ -29,14 +29,29 @@ class AtokaOwnershipTransformation(Transformation):
                 'founded': owned_founded,
                 'atoka_id': owned_atoka_id,
                 'tax_id': owned_tax_id,
-                'rea': owner_rea_id,
-                'cciaa': owner_cciaa,
+                'rea': owned_rea_id,
+                'cciaa': owned_cciaa,
                 'legal_form': owned_atoka_legal_form_level2,
                 'ateco': owned_ateco,
                 'percentage': ratio * 100,
                 'last_update': share_last_update
             },
             ...
+        ],
+        'shareholders': [
+            {
+                'name': owner_name,
+                'founded': owner_founded,
+                'atoka_id': owner_atoka_id,
+                'tax_id': owner_tax_id,
+                'rea': owner_rea_id,
+                'cciaa': owner_cciaa,
+                'legal_form': owner_atoka_legal_form_level2,
+                'ateco': owner_ateco,
+                'percentage': ratio * 100,
+                'last_update': share_last_update
+            },
+            ...
         ]
     },
     ...
@@ -106,6 +121,27 @@ class AtokaOwnershipTransformation(Transformation):
 
                     _ownerships.append(_ownership)
 
+            for owner in i['shareholders']:
+                owning_org_tax_id = owner.get("tax_id", None)
+                owned_org_tax_id = i.get("tax_id", None)
+                if owning_org_tax_id and owned_org_tax_id:
+                    _ownership = {
+                        "owning_org": {
+                            "identifier": owning_org_tax_id,
+                        },
+                        "owned_org": {
+                            "identifier": owned_org_tax_id,
+                        },
+                        "percentage": owner["percentage"],
+                        "sources": [{
+                            "url": "https://api.atoka.io",
+                            "note": "ATOKA API"
+                        }]
+
+                    }
+
+                    _ownerships.append(_ownership)
+
             return _ownerships
 
         def get_index(o: dict) -> tuple:
@@ -150,8 +186,23 @@ class AtokaOrganizationTransformation(Transformation):
                 'founded': owned_founded,
                 'atoka_id': owned_atoka_id,
                 'tax_id': owned_tax_id,
-                'rea': owner_rea_id,
-                'cciaa': owner_cciaa,
+                'rea': owned_rea_id,
+                'cciaa': owned_cciaa,
+                'legal_form': owned_atoka_legal_form_level2,
+                'ateco': owned_ateco,
+                'percentage': ratio * 100,
+                'last_update': share_last_update
+            },
+            ...
+        ],
+        'shareholders': [
+            {
+                'name': owned_name,
+                'founded': owned_founded,
+                'atoka_id': owned_atoka_id,
+                'tax_id': owned_tax_id,
+                'rea': owned_rea_id,
+                'cciaa': owned_cciaa,
                 'legal_form': owned_atoka_legal_form_level2,
                 'ateco': owned_ateco,
                 'percentage': ratio * 100,
@@ -372,11 +423,11 @@ class AtokaOrganizationTransformation(Transformation):
             if index and index not in unique_set:
                 self.etl.processed_data.append(owning_org)
                 unique_set.add(index)
-                for owned_item in owning_item["shares_owned"]:
-                    owned_org = org_from_item(owned_item)
-                    index = get_index(owned_org)
+                for item in owning_item["shares_owned"] + owning_item['shareholders']:
+                    org = org_from_item(item)
+                    index = get_index(org)
                     if index and index not in unique_set:
-                        self.etl.processed_data.append(owned_org)
+                        self.etl.processed_data.append(org)
                         unique_set.add(index)
 
 

setup.cfg

 [bumpversion]
-current_version = 1.1.9
+current_version = 1.1.10
 commit = True
 tag = True
 tag_name = v{new_version}