CHANGELOG.md

@@ -5,10 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.1.11]
+- transformation producing organizations json file corrected;
+  owned orgs and shareholders added also when main org already added;
+  this avoids missing organizations log messages;
+- fullAddress property from atoka imported into all organizations, as
+  contact_detail of type MAIL
+
 ## [1.1.10]
 
 ### Add
-- shareholders importe along with sharesOwned from ATOKA
+- shareholders importes along with sharesOwned from ATOKA
 
 ## [1.1.9]
 

project/__init__.py

@@ -3,4 +3,4 @@
 Openpolis Data Manager service package (backend)
 """
 
-__version__ = '1.1.10'
+__version__ = '1.1.11'

project/api_v1/etl/loaders/organizations.py

@@ -58,10 +58,10 @@ class PopoloOrgOwnershipLoader(PopoloLoader):
             logger=self.logger
         )
         if owner_org_id == 0:
-            self.logger.error("Could not find owner org in {0}".format(item["owner_org"]))
+            self.logger.error("Could not find owner org in {0}".format(item["owning_org"]))
             return
         elif owner_org_id < 0:
-            self.logger.error("More than one owner org found for tax_id {0}".format(item["owner_org"]))
+            self.logger.error("More than one owner org found for tax_id {0}".format(item["owning_org"]))
             return
         else:
             try:

project/atoka/etl/extractors.py

@@ -45,14 +45,16 @@ class AtokaOwnershipsExtractor(Extractor):
                 'legal_form': owner_atoka_legal_form_level2,
                 'rea': owner_rea_id,
                 'cciaa': owner_cciaa,
+                'full_address': owner_full_address,
                 'shares_owned': [
                     {
                         'name': owned_name,
                         'founded': owned_founded,
                         'atoka_id': owned_atoka_id,
                         'tax_id': owned_tax_id,
-                        'rea': owner_rea_id,
-                        'cciaa': owner_cciaa,
+                        'rea': owned_rea_id,
+                        'cciaa': owned_cciaa,
+                        'full_address': owned_full_address,
                         'legal_form': owned_atoka_legal_form_level2,
                         'ateco': owned_ateco,
                         'percentage': ratio * 100,
@@ -158,6 +160,7 @@ class AtokaOwnershipsExtractor(Extractor):
                     'legal_form': [x['name'] for x in r['base']['legalForms'] if x['level'] == 2][0],
                     'rea': r['base'].get('rea', None),
                     'cciaa': r['base'].get('cciaa', None),
+                    'full_address': r.get('fullAddress', None),
                     'shares_owned': [],
                     'shareholders':  [],
                 }
@@ -172,7 +175,9 @@ class AtokaOwnershipsExtractor(Extractor):
                     'percentage': sho.get('ratio', 0.) * 100.
                 }
                 for sho in filter(
-                    lambda x: x['active'] is True and x['typeOfRight'] == 'proprietà',
+                    lambda x:
+                        x.get('active', False) is True and
+                        x.get('typeOfRight', None) == 'proprietà',
                     r['shares']['sharesOwned']
                 )
             )
@@ -184,7 +189,9 @@ class AtokaOwnershipsExtractor(Extractor):
                     'percentage': sho.get('ratio', 0.) * 100.
                 }
                 for sho in filter(
-                    lambda x: x['active'] is True and x['typeOfRight'] == 'proprietà',
+                    lambda x:
+                        x.get('active', False) is True and
+                        x.get('typeOfRight', None) == 'proprietà',
                     r['shares'].get('shareholders', [])
                 )
             )
@@ -209,6 +216,7 @@ class AtokaOwnershipsExtractor(Extractor):
                 'tax_id': r['base'].get('taxId', None),
                 'vat': r['base'].get('vat', None),
                 'founded': r['base'].get('founded', None),
+                'full_address': r.get('fullAddress', None),
                 'legal_form': [x['name'] for x in r['base']['legalForms'] if x['level'] == 2][0],
                 'ateco': r['base'].get('ateco', [])
             } for r in owned_orgs
@@ -233,6 +241,7 @@ class AtokaOwnershipsExtractor(Extractor):
                 'tax_id': r['base'].get('taxId', None),
                 'vat': r['base'].get('vat', None),
                 'founded': r['base'].get('founded', None),
+                'full_address': r.get('fullAddress', None),
                 'legal_form': [x['name'] for x in r['base']['legalForms'] if x['level'] == 2][0],
                 'ateco': r['base'].get('ateco', [])
             } for r in owners_orgs
@@ -326,7 +335,10 @@ class AtokaOwnershipsExtractor(Extractor):
             for owned in org['shares_owned']:
                 owned_details = owned_orgs_dict.get(owned['atoka_id'], None)
                 if owned_details:
-                    for f in ['name', 'cciaa', 'rea', 'ateco', 'tax_id', 'vat', 'founded', 'legal_form', 'roles']:
+                    for f in [
+                        'name', 'cciaa', 'rea', 'ateco', 'tax_id', 'vat',
+                        'founded', 'legal_form', 'full_address', 'roles'
+                    ]:
                         if f in owned_details:
                             owned[f] = owned_details[f]
                 else:
@@ -337,7 +349,9 @@ class AtokaOwnershipsExtractor(Extractor):
             for owner in org['shareholders']:
                 owner_details = owner_orgs_dict.get(owner['atoka_id'], None)
                 if owner_details:
-                    for f in ['name', 'cciaa', 'rea', 'ateco', 'tax_id', 'vat', 'founded', 'legal_form']:
+                    for f in [
+                        'name', 'cciaa', 'rea', 'ateco', 'tax_id', 'vat', 'founded', 'legal_form', 'full_address'
+                    ]:
                         if f in owner_details:
                             owner[f] = owner_details[f]
                 else:

project/atoka/etl/transformations.py

@@ -357,6 +357,7 @@ class AtokaOrganizationTransformation(Transformation):
                     "name": i["name"],
                     "foundation_date": i.get("founded", None),
                     "classifications": [],
+                    "contact_details": [],
                     "identifiers": [
                         {
                             "scheme": "ATOKA_ID",
@@ -399,6 +400,13 @@ class AtokaOrganizationTransformation(Transformation):
                             "url": "https://api.atoka.io/v2/companies/{0}".format(l),
                             "note": "Altra company in ATOKA con stesso CF"
                         })
+                if "full_address" in i and i["full_address"]:
+                    o["contact_details"].append({
+                        "label": "Indirizzo della sede legale",
+                        "contact_type": "MAIL",
+                        "value": i["full_address"],
+                        "note": "Indirizzo da Atoka"
+                    })
                 return o
 
         # store processed data into the Transformation instance
@@ -423,12 +431,12 @@ class AtokaOrganizationTransformation(Transformation):
             if index and index not in unique_set:
                 self.etl.processed_data.append(owning_org)
                 unique_set.add(index)
-                for item in owning_item["shares_owned"] + owning_item['shareholders']:
-                    org = org_from_item(item)
-                    index = get_index(org)
-                    if index and index not in unique_set:
-                        self.etl.processed_data.append(org)
-                        unique_set.add(index)
+            for item in owning_item["shares_owned"] + owning_item['shareholders']:
+                org = org_from_item(item)
+                index = get_index(org)
+                if index and index not in unique_set:
+                    self.etl.processed_data.append(org)
+                    unique_set.add(index)
 
 
 class AtokaMembershipTransformation(Transformation):

project/atoka/management/commands/import_atoka.py

@@ -23,7 +23,7 @@ class Command(LoggingBaseCommand):
         parser.add_argument(
             "--data-path",
             dest="datapath",
-            default="./resources/data/atoka",
+            default="./data/atoka",
             help="Complete path to json files"
         )
         parser.add_argument(

project/atoka/management/commands/import_atoka_economics.py

@@ -18,7 +18,7 @@ class Command(LoggingBaseCommand):
         parser.add_argument(
             "--data-path",
             dest="datapath",
-            default="./resources/data/atoka",
+            default="./data/atoka",
             help="Complete path to json files"
         )
 

project/atoka/management/commands/import_atoka_economics_extract_json.py

@@ -27,7 +27,7 @@ class Command(LoggingBaseCommand):
         parser.add_argument(
             "--json-file",
             dest="jsonfile",
-            default="./resources/data/atoka/atoka_economics.json",
+            default="./data/atoka/atoka_economics.json",
             help="Complete path to json file"
         )
 

project/atoka/management/commands/import_atoka_extract_json.py

@@ -41,7 +41,7 @@ class Command(LoggingBaseCommand):
         parser.add_argument(
             "--json-file",
             dest="jsonfile",
-            default="./resources/data/atoka/atoka.json",
+            default="./data/atoka/atoka.json",
             help="Complete path to json file"
         )
         parser.add_argument(

project/atoka/management/commands/import_atoka_meta.py

@@ -128,7 +128,7 @@ class Command(LoggingBaseCommand):
         parser.add_argument(
             "--data-path",
             dest="datapath",
-            default="./resources/data/atoka",
+            default="./data/atoka",
             help="Complete path to json files"
         )
         parser.add_argument(

project/atoka/management/commands/import_atoka_transform_json.py

@@ -19,13 +19,13 @@ class Command(LoggingBaseCommand):
         parser.add_argument(
             "--json-source-file",
             dest="json_src",
-            default="./resources/data/atoka/atoka.json",
+            default="./data/atoka/atoka.json",
             help="Complete path to source file with atoka info"
         )
         parser.add_argument(
             "--json-output-path",
             dest="json_out_path",
-            default="./resources/data/atoka",
+            default="./data/atoka",
             help="Path to directory where emitted json files will be stored"
         )
         parser.add_argument(

setup.cfg

 [bumpversion]
-current_version = 1.1.10
+current_version = 1.1.11
 commit = True
 tag = True
 tag_name = v{new_version}