CHANGELOG.md

@@ -5,10 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+
+## [1.1.12]
+
+### Fixed
+- shareholders imported together with sharesOwned organizations;
+  shares-level=0 import all level-1 orgs, and their shareholders;
+
 ## [1.1.11]
+
+### Fixed
 - transformation producing organizations json file corrected;
   owned orgs and shareholders added also when main org already added;
   this avoids missing organizations log messages;
+
+### Add
 - fullAddress property from atoka imported into all organizations, as
   contact_detail of type MAIL
 

project/__init__.py

@@ -3,4 +3,4 @@
 Openpolis Data Manager service package (backend)
 """
 
-__version__ = '1.1.11'
+__version__ = '1.1.12'

project/atoka/etl/extractors.py

@@ -162,7 +162,6 @@ class AtokaOwnershipsExtractor(Extractor):
                     'cciaa': r['base'].get('cciaa', None),
                     'full_address': r.get('fullAddress', None),
                     'shares_owned': [],
-                    'shareholders':  [],
                 }
             else:
                 r_dict = res_dict[r['base']['taxId']]
@@ -181,7 +180,33 @@ class AtokaOwnershipsExtractor(Extractor):
                     r['shares']['sharesOwned']
                 )
             )
-            r_dict['shareholders'].extend(
+
+            res_dict[r['base']['taxId']] = r_dict
+
+        # extract all atoka_ids from shares_owned elements and returns flat list
+        # then apply list(set(x)) to remove duplicates, if any
+        owned_atoka_ids = list(set(list(itertools.chain.from_iterable([
+            [x['atoka_id'] for x in r['shares_owned']]
+            for r in res_dict.values()
+        ]))))
+
+        owned_orgs = atoka_conn.get_companies_from_atoka_ids(
+            owned_atoka_ids, packages='base,shares', active='true', batch_size=10
+        )
+
+        # build the list of all atoka_ids of owners at level shares_level
+        # used while building list of shareholders, and exclude main owner
+        atoka_ids = [
+            r['atoka_id'] for r in res_dict.values()
+        ] + list(
+            itertools.chain.from_iterable(
+                [r['other_atoka_ids'] for r in res_dict.values()]
+            )
+        )
+
+        owned_orgs_dict = {}
+        for r in owned_orgs:
+            shareholders = [
                 {
                     'name': sho['name'],
                     'last_updated': sho['lastUpdate'],
@@ -194,22 +219,10 @@ class AtokaOwnershipsExtractor(Extractor):
                         x.get('typeOfRight', None) == 'proprietà',
                     r['shares'].get('shareholders', [])
                 )
-            )
-            res_dict[r['base']['taxId']] = r_dict
-
-        # extract all atoka_ids from shares_owned elements and returns flat list
-        # then apply list(set(x)) to remove duplicates, if any
-        owned_atoka_ids = list(set(list(itertools.chain.from_iterable([
-            [x['atoka_id'] for x in r['shares_owned']]
-            for r in res_dict.values()
-        ]))))
+                if sho['id'] not in atoka_ids
+            ]
 
-        owned_orgs = atoka_conn.get_companies_from_atoka_ids(
-            owned_atoka_ids, packages='base', active='true', batch_size=10
-        )
-
-        owned_orgs_dict = {
-            r['id']: {
+            owned_orgs_dict[r['id']] = {
                 'name': r['name'],
                 'cciaa': r['base'].get('cciaa', None),
                 'rea': r['base'].get('rea', None),
@@ -218,22 +231,22 @@ class AtokaOwnershipsExtractor(Extractor):
                 'founded': r['base'].get('founded', None),
                 'full_address': r.get('fullAddress', None),
                 'legal_form': [x['name'] for x in r['base']['legalForms'] if x['level'] == 2][0],
-                'ateco': r['base'].get('ateco', [])
-            } for r in owned_orgs
-        }
+                'ateco': r['base'].get('ateco', []),
+                'shareholders': shareholders
+            }
 
         # extract all atoka_ids from shareholders elements and returns flat list
         # then apply list(set(x)) to remove duplicates, if any
-        owners_atoka_ids = list(set(list(itertools.chain.from_iterable([
+        shareholders_atoka_ids = list(set(list(itertools.chain.from_iterable([
             [x['atoka_id'] for x in r['shareholders']]
-            for r in res_dict.values()
+            for r in owned_orgs_dict.values()
         ]))))
 
-        owners_orgs = atoka_conn.get_companies_from_atoka_ids(
-            owners_atoka_ids, packages='base', active='true', batch_size=10
+        shareholders_orgs = atoka_conn.get_companies_from_atoka_ids(
+            shareholders_atoka_ids, packages='base', active='true', batch_size=10
         )
 
-        owner_orgs_dict = {
+        shareholders_orgs_dict = {
             r['id']: {
                 'name': r['name'],
                 'cciaa': r['base'].get('cciaa', None),
@@ -244,11 +257,11 @@ class AtokaOwnershipsExtractor(Extractor):
                 'full_address': r.get('fullAddress', None),
                 'legal_form': [x['name'] for x in r['base']['legalForms'] if x['level'] == 2][0],
                 'ateco': r['base'].get('ateco', [])
-            } for r in owners_orgs
+            } for r in shareholders_orgs
         }
 
-        atoka_companies_requests += len(owned_orgs) + len(owners_orgs)
-        self.logger.debug("- ricevuti dettagli per {0} partecipate".format(len(owned_orgs) + len(owners_orgs)))
+        atoka_companies_requests += len(owned_orgs) + len(shareholders_orgs)
+        self.logger.debug("- ricevuti dettagli per {0} partecipate".format(len(owned_orgs) + len(shareholders_orgs)))
 
         # extract all people's atoka_ids from res_owned elements and returns flat list, removing duplicates
         people = atoka_conn.get_roles_from_atoka_ids(
@@ -337,28 +350,27 @@ class AtokaOwnershipsExtractor(Extractor):
                 if owned_details:
                     for f in [
                         'name', 'cciaa', 'rea', 'ateco', 'tax_id', 'vat',
-                        'founded', 'legal_form', 'full_address', 'roles'
+                        'founded', 'legal_form', 'full_address', 'roles', 'shareholders'
                     ]:
                         if f in owned_details:
                             owned[f] = owned_details[f]
+
+                    if 'shareholders' in owned_details:
+                        for shareholder in owned_details['shareholders']:
+                            shareholder_details = shareholders_orgs_dict.get(shareholder['atoka_id'], None)
+                            if shareholder_details:
+                                for f in [
+                                    'name', 'cciaa', 'rea', 'ateco', 'tax_id', 'vat',
+                                    'founded', 'legal_form', 'full_address'
+                                ]:
+                                    if f in shareholder_details:
+                                        shareholder[f] = shareholder_details[f]
+
                 else:
                     self.logger.warning(
                         "! organizzazione {0} richiesta ad atoka, "
                         "ma non presente nei risultati".format(owned['atoka_id'])
                     )
-            for owner in org['shareholders']:
-                owner_details = owner_orgs_dict.get(owner['atoka_id'], None)
-                if owner_details:
-                    for f in [
-                        'name', 'cciaa', 'rea', 'ateco', 'tax_id', 'vat', 'founded', 'legal_form', 'full_address'
-                    ]:
-                        if f in owner_details:
-                            owner[f] = owner_details[f]
-                else:
-                    self.logger.warning(
-                        "! organizzazione {0} richiesta ad atoka, "
-                        "ma non presente nei risultati".format(owner['atoka_id'])
-                    )
 
         # returns a list
         results = []

project/atoka/etl/transformations.py

@@ -52,7 +52,7 @@ class AtokaOwnershipTransformation(Transformation):
                 'last_update': share_last_update
             },
             ...
-        ]
+        ],
     },
     ...
 
@@ -121,26 +121,26 @@ class AtokaOwnershipTransformation(Transformation):
 
                     _ownerships.append(_ownership)
 
-            for owner in i['shareholders']:
-                owning_org_tax_id = owner.get("tax_id", None)
-                owned_org_tax_id = i.get("tax_id", None)
-                if owning_org_tax_id and owned_org_tax_id:
-                    _ownership = {
-                        "owning_org": {
-                            "identifier": owning_org_tax_id,
-                        },
-                        "owned_org": {
-                            "identifier": owned_org_tax_id,
-                        },
-                        "percentage": owner["percentage"],
-                        "sources": [{
-                            "url": "https://api.atoka.io",
-                            "note": "ATOKA API"
-                        }]
+                for shareholder in owned['shareholders']:
+                    owning_org_tax_id = shareholder.get("tax_id", None)
+                    owned_org_tax_id = owned.get("tax_id", None)
+                    if owning_org_tax_id and owned_org_tax_id:
+                        _ownership = {
+                            "owning_org": {
+                                "identifier": owning_org_tax_id,
+                            },
+                            "owned_org": {
+                                "identifier": owned_org_tax_id,
+                            },
+                            "percentage": shareholder["percentage"],
+                            "sources": [{
+                                "url": "https://api.atoka.io",
+                                "note": "ATOKA API"
+                            }]
 
-                    }
+                        }
 
-                    _ownerships.append(_ownership)
+                        _ownerships.append(_ownership)
 
             return _ownerships
 
@@ -191,25 +191,25 @@ class AtokaOrganizationTransformation(Transformation):
                 'legal_form': owned_atoka_legal_form_level2,
                 'ateco': owned_ateco,
                 'percentage': ratio * 100,
-                'last_update': share_last_update
+                'last_update': share_last_update,
+                'shareholders': [
+                    {
+                        'name': owned_name,
+                        'founded': owned_founded,
+                        'atoka_id': owned_atoka_id,
+                        'tax_id': owned_tax_id,
+                        'rea': owned_rea_id,
+                        'cciaa': owned_cciaa,
+                        'legal_form': owned_atoka_legal_form_level2,
+                        'ateco': owned_ateco,
+                        'percentage': ratio * 100,
+                        'last_update': share_last_update
+                    },
+                    ...
+                ]
             },
             ...
         ],
-        'shareholders': [
-            {
-                'name': owned_name,
-                'founded': owned_founded,
-                'atoka_id': owned_atoka_id,
-                'tax_id': owned_tax_id,
-                'rea': owned_rea_id,
-                'cciaa': owned_cciaa,
-                'legal_form': owned_atoka_legal_form_level2,
-                'ateco': owned_ateco,
-                'percentage': ratio * 100,
-                'last_update': share_last_update
-            },
-            ...
-        ]
     },
     ...
 
@@ -431,12 +431,18 @@ class AtokaOrganizationTransformation(Transformation):
             if index and index not in unique_set:
                 self.etl.processed_data.append(owning_org)
                 unique_set.add(index)
-            for item in owning_item["shares_owned"] + owning_item['shareholders']:
+            for item in owning_item["shares_owned"]:
                 org = org_from_item(item)
                 index = get_index(org)
                 if index and index not in unique_set:
                     self.etl.processed_data.append(org)
                     unique_set.add(index)
+                for shareholder_item in item['shareholders']:
+                    shareholder_org = org_from_item(shareholder_item)
+                    index = get_index(shareholder_org)
+                    if index and index not in unique_set:
+                        self.etl.processed_data.append(shareholder_org)
+                        unique_set.add(index)
 
 
 class AtokaMembershipTransformation(Transformation):

setup.cfg

 [bumpversion]
-current_version = 1.1.11
+current_version = 1.1.12
 commit = True
 tag = True
 tag_name = v{new_version}