.gitignore

@@ -48,6 +48,7 @@ dump*
 
 *.zip
 
+/data
 resources/data/bdap_anagrafe_enti.csv
 resources/data/parsers
 resources/data/out

CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [1.1.2]
+
+### Fixed
+- founding_date is imported correctly
+- double FORMA_GIURIDICA_OP classification is avoided when importing from ATOKA
+- RoleTypes generation from atoka mapping corrected
+- Commissario straordinario removed from atoka_roles_skip_list
+
+### Changed
+- CCIAA classification and REA identifier from ATOKA moved into one CCIAA-REA identifier
+- It is now impossible to add more than one classifications of the same scheme to an object (popolo).
+
 ## [1.1.1]
 
 ### Added
@@ -14,7 +26,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Changed
 - is_public flag (SPA) and capital_stock added to OrganizationEconomics
-
+- json files for organizations can specify classifications by: 
+  classification (id), scheme/code, scheme/descr, scheme/code/descr (non-existing)
+     
 ## [1.1.0]
 
 ### Fixed

project/__init__.py

@@ -3,4 +3,4 @@
 Openpolis Data Manager service package (backend)
 """
 
-__version__ = '1.1.1'
+__version__ = '1.1.2'

project/api_v1/management/commands/script_extract_roles_from_atoka.py

@@ -8,7 +8,7 @@ from django.core.management import BaseCommand
 from popolo.models import Organization
 
 from project.api_v1.core import batch_generator
-from project.connections.atoka import AtokaConn, AtokaObjectDoesNotExist
+from project.atoka.connections import AtokaConn, AtokaObjectDoesNotExist
 
 
 class Command(BaseCommand):

project/api_v1/management/commands/script_generate_role_types_mapping.py

@@ -20,6 +20,12 @@ class Command(BaseCommand):
             default="./resources/data/role_types_atoka_opdm.csv",
             help="Complete csv filename",
         )
+        parser.add_argument(
+            "--delete",
+            dest="delete",
+            action='store_true',
+            help="Reset by deleting all roletypes that would be created (in development phase)"
+        )
 
     def handle(self, *args, **options):
         verbosity = options["verbosity"]
@@ -41,19 +47,28 @@ class Command(BaseCommand):
             if n == 0:
                 classifications_mapping = row
             else:
-                for k, v in classifications_mapping.items():
-                    if v is np.nan:
+                for atoka_label, opdm_id in classifications_mapping.items():
+                    if opdm_id is np.nan:
                         continue
-                    v = int(v)
-                    if row[k] and v > 0:
-                        rt_label = "{0} {1}".format(
-                            row['role_type_opdm'],
-                            Classification.objects.get(id=v).descr.lower()
-                        )
-                        rt, created = RoleType.objects.get_or_create(
+                    opdm_id = int(opdm_id)
+                    rt_label = "{0} {1}".format(
+                        row['role_type_opdm'],
+                        Classification.objects.get(id=opdm_id).descr.lower()
+                    )
+                    if options['delete']:
+                        rt = RoleType.objects.filter(
                             label=rt_label,
-                            classification_id=v
+                            classification_id=opdm_id
                         )
-                        if created:
-                            print(rt)
+                        if rt:
+                            rt.delete()
+                            print(rt_label + " deleted")
+                    else:
+                        if int(row[atoka_label]) > 0:
+                            rt, created = RoleType.objects.get_or_create(
+                                label=rt_label,
+                                classification_id=opdm_id
+                            )
+                            if created:
+                                print(rt_label + " created")
         self.logger.info("End")

project/api_v1/serializers/inlines.py

@@ -12,9 +12,12 @@ from project.api_v1.serializers import GenericRelatableSerializer
 
 
 class ClassificationInlineSerializer(serializers.ModelSerializer):
+    code = serializers.CharField(required=False)
+    descr = serializers.CharField(required=False)
+
     def run_validation(self, data=empty):
         """
-        We override the default `run_validation`, and intercept the case in which `id`.
+        We override the default `run_validation`, and intercept the case in which `id` is passed.
 
         This use case means that the classification we want to associate already exists and the UniqueTogetherValidator,
         should not report an exception (duplicates are avoided by the get_or_create method used to add the
@@ -27,6 +30,10 @@ class ClassificationInlineSerializer(serializers.ModelSerializer):
         if 'classification' in data and data['classification']:
             return data
 
+        # validation passes if one of 'code' and 'descr' has value; 'scheme' must always be specified
+        if data.get('scheme', None) and (data.get('code', None) or data.get('descr', None)):
+            return data
+
         return super().run_validation(data)
 
     class Meta:

project/api_v1/tests/test_organization.py

@@ -111,10 +111,15 @@ class OrganizationTestCase(
 
         :return:
         """
+        classification_a = ClassificationFactory()
+        classification_b = ClassificationFactory()
+
+        # test various methods to add a classification
         classifications = [
-            {"classification": ClassificationFactory().id},
-            {"classification": ClassificationFactory().id},
-            {"classification": ClassificationFactory().id},
+            {"classification": classification_a.id},                               # existing classification id
+            {"scheme": faker.word(), "code": faker.ssn(), "descr": faker.word()},  # non-existing classification by dict
+            {"scheme": classification_b.scheme, "code": classification_b.code},    # existing classification by dict
+            {"scheme": classification_b.scheme, "code": faker.word()}              # no duplication of same-scheme-class
         ]
 
         org = {
@@ -130,11 +135,6 @@ class OrganizationTestCase(
         r = json.loads(response.content)
         self.assertEquals(len(r["classifications"]), 3)
 
-        self.assertEquals(
-            sorted([c["classification"] for c in classifications]),
-            sorted([c["classification"]["id"] for c in r["classifications"]]),
-        )
-
     def test_create_organizations_classifications_donot_repeat(self):
         """ Test creation of 2 organizations with the same 2 classifications.
 

project/atoka/etl/transformations.py

@@ -176,14 +176,10 @@ class AtokaOrganizationTransformation(Transformation):
             },
             {
                 "scheme": "REA",
-                "identifier": org_rea
+                "identifier": "{org_cia}-{org_rea}"
             }
         ],
         "classifications": [
-            {
-                "scheme": "CCIAA",
-                "descr": org_cciaa
-            },
             {
                 "scheme": "FORMA_GIURIDICA_OP",
                 "descr": org_legal_form
@@ -307,7 +303,7 @@ class AtokaOrganizationTransformation(Transformation):
                 o = {
                     "identifier": identifier,
                     "name": i["name"],
-                    "foundation_date": i.get("founding", None),
+                    "foundation_date": i.get("founded", None),
                     "classifications": [],
                     "identifiers": [
                         {
@@ -328,21 +324,16 @@ class AtokaOrganizationTransformation(Transformation):
                     })
                     o["classification"] = classifications_dict[classification_id]
 
-                if "rea" in i and i['rea']:
+                if "rea" in i and i['rea'] and "cciaa" in i and i['cciaa']:
                     o["identifiers"].append({
-                        "scheme": "REA",
-                        "identifier": i["rea"]
+                        "scheme": "CCIAA-REA",
+                        "identifier": "{0}-{1}".format(i["cciaa"], i["rea"])
                     })
                 if "vat" in i and i['vat']:
                     o["identifiers"].append({
                         "scheme": "VAT",
                         "identifier": i["vat"]
                     })
-                if "cciaa" in i and i['cciaa']:
-                    o["classifications"].append({
-                        "scheme": "CCIAA",
-                        "descr": i["cciaa"]
-                    })
                 if "other_atoka_ids" in i and len(i["other_atoka_ids"]):
                     for l in i["other_atoka_ids"]:
                         o["links"].append({
@@ -496,7 +487,6 @@ class AtokaMembershipTransformation(Transformation):
         atoka_roles_skip_list = [
             "board member",
             "board members",
-            "commissario straordinario",
             "delegato",
             "direttore o manager",
             "membro comitato direttivo",

project/atoka/tests/etl/test_atoka.py

@@ -75,7 +75,7 @@ class AtokaETLTest(SolrETLTest):
         """
 
         # mock atoka_conn method for cciaa or govTypes results
-        AtokaConn.get_companies_from_tax_ids = MagicMock(
+        AtokaConn.get_companies_from_atoka_ids = MagicMock(
             side_effect=get_companies_economics_extractor,
             status_code=200,
             ok=True
@@ -83,13 +83,13 @@ class AtokaETLTest(SolrETLTest):
 
         # extract info
         res = AtokaEconomicsExtractor(
-            ['02241890223', '09988761004']
+            ['6da785b3adf2', '38e098baa0f9']
         ).extract()['results']
 
         self.assertEqual(len(res), 2)
 
         # test complete economics info
-        c = res[next(i for i, v in enumerate(res) if v['tax_id'] == '09988761004')]
+        c = res[next(i for i, v in enumerate(res) if v['id'] == '38e098baa0f9')]
         self.assertEqual(c['id'], '38e098baa0f9')
         self.assertEqual(c['name'], 'DEPP SRL')
         ce = c['economics']
@@ -132,7 +132,7 @@ class AtokaETLTest(SolrETLTest):
         self.assertEqual(sh['profit'], 12000)
         self.assertEqual(sh['staff_costs'], 157000)
         self.assertEqual(sh['purchases'], 0)
-        self.assertEqual(sh['employees'], 9)
+        self.assertEqual(sh['employees'], 7)
         self.assertEqual(sh['services_and_tp_goods_charges'], 285000)
         self.assertEqual(sh['net_financial_position'], 7000)
 
@@ -148,4 +148,4 @@ class AtokaETLTest(SolrETLTest):
         sh = c['historical_values'][0]
         self.assertEqual(sh.get('assets', None), None)
         self.assertEqual(sh.get('costs', None), None)
-        self.assertEqual(sh['employees'], 9)
+        self.assertEqual(sh['employees'], 26)

setup.cfg

 [bumpversion]
-current_version = 1.1.1
+current_version = 1.1.2
 commit = True
 tag = True
 tag_name = v{new_version}