CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [1.0.2]
+
+### Changed
+
+- `--use-dummy-transformation` added to import_persons_memberships_from_json task
+  to keeb back compatibility with national level imports and use it in
+  new atoka imports
+- `--log-step` and `--context` parameters added to import_persons_memberships_from_json
+  to allow for setting context string in loader_context (similarities)
+
 
 ## [1.0.1]
 

project/__init__.py

@@ -3,4 +3,4 @@
 Openpolis Data Manager service package (backend)
 """
 
-__version__ = '1.0.1'
+__version__ = '1.0.2'

project/api_v1/core.py

@@ -1228,8 +1228,8 @@ class PersonUtils(object):
                             search_params[k] = v.lower()
                     else:
                         search_params[k] = ""
-            if logger:
-                logger.debug(search_params)
+            # if logger:
+            #     logger.debug(search_params)
         elif strategy == "identifier":
             # identifier lookup strategy: use given scheme or lookup on first identifier
             if identifier_scheme:

project/api_v1/etl/loaders/organizations.py

@@ -32,7 +32,7 @@ class PopoloOrgOwnershipLoader(PopoloLoader):
         """
         super(PopoloOrgOwnershipLoader, self).__init__(**kwargs)
         self.lookup_strategy = kwargs.get('lookup_strategy', 'mixed')
-        self.identifier_scheme = kwargs.get('identifier_scheme', 'CODICE_ENTE_BDAP')
+        self.identifier_scheme = kwargs.get('identifier_scheme', None)
         self.update_strategy = kwargs.get('update_strategy', 'keep_old')
 
     def load_item(self, item, **kwargs):
@@ -60,21 +60,27 @@ class PopoloOrgOwnershipLoader(PopoloLoader):
         if owner_org_id == 0:
             self.logger.error("Could not find owner org in {0}".format(item["owner_org"]))
             return
+        elif owner_org_id < 0:
+            self.logger.error("More than one owner org found for tax_id {0}".format(item["owner_org"]))
+            return
         else:
             try:
                 owner_org = Organization.objects.get(id=owner_org_id)
             except Organization.DoesNotExist:
-                self.logger.error("Organization ID found but Organization missing: {0}".format(owner_org_id))
+                self.logger.error("Owner organization ID found but Organization missing: {0}".format(owner_org_id))
                 return
 
         if owned_org_id == 0:
             self.logger.error("Could not find owned org in {0}".format(item["owned_org"]))
             return
+        elif owned_org_id < 0:
+            self.logger.error("More than one owned org found for tax_id {0}".format(item["owned_org"]))
+            return
         else:
             try:
                 owned_org = Organization.objects.get(id=owned_org_id)
             except Organization.DoesNotExist:
-                self.logger.error("Organization ID found but Organization missing: {0}".format(owned_org_id))
+                self.logger.error("Owner organization ID found but Organization missing: {0}".format(owned_org_id))
                 return
 
         try:

project/api_v1/etl/transformations/json2opdm.py

@@ -15,9 +15,7 @@ class JsonPersonsMemberships2OpdmTransformation(Transformation):
     def transform(self):
         """ Transform a list of dicts parsed from the JSON file,
         annotating the memberships with electoral KeyEvent, Organization, and RoleType instances,
-        in order to re-use the
-
-        :return: the ETL instance (to chain methods)
+        in order to re-use the PersonsWithMemberships loader
         """
         self.logger.info("start of transform")
         classification_labels = [

project/api_v1/management/commands/import_atoka_transform_json.py

@@ -62,7 +62,7 @@ class Command(LoggingBaseCommand):
         ETL(
             extractor=ListExtractor(atoka_records),
             transformation=AtokaMembershipTransformation(),
-            loader=JsonLoader(os.path.join(json_out_path, "atoka_person_memberships.json")),
+            loader=JsonLoader(os.path.join(json_out_path, "atoka_persons_memberships.json")),
             log_level=self.logger.level,
             # source="http://api.atoka.it"
         )()

project/api_v1/management/commands/import_persons_memberships_from_json.py

 import logging
 
-from opdmetl import ETL
+from opdmetl import ETL, DummyTransformation
 from taskmanager.utils import LoggingBaseCommand
 
 from project.api_v1.etl.loaders.persons import PopoloPersonWithMembershipsLoader
@@ -32,12 +32,31 @@ class Command(LoggingBaseCommand):
             help="Whether to keep old values or to overwrite them "
             "for Membership updates (keep_old | overwrite | overwrite_minint_opdm), defaults to overwrite_minint_opdm",
         )
+        parser.add_argument(
+            "--use-dummy-transformation",
+            dest="use_dummy_transformation",
+            action="store_true",
+            help="Whether to use the DummyTransformation instead of JsonPersonsMemberships2OpdmTransformation",
+        )
         parser.add_argument(
             "--check-membership-label",
             dest="check_membership_label",
             action="store_true",
             help="Whether to check membership labels when updating",
         )
+        parser.add_argument(
+            "--log-step",
+            dest="log_step",
+            type=int,
+            default=50,
+            help="Number of steps to log process completion to stdout. Defaults to 500.",
+        )
+        parser.add_argument(
+            "--context",
+            dest="context",
+            default='parlamento_it',
+            help="Context string used in loader_context",
+        )
 
     def handle(self, *args, **options):
         self.setup_logger(__name__, formatter_key="simple", **options)
@@ -48,18 +67,26 @@ class Command(LoggingBaseCommand):
         memberships_update_strategy = options["memberships_update_strategy"]
         check_membership_label = options["check_membership_label"]
         source_url = options["source_url"]
+        log_step = options["log_step"]
+        context = options["context"]
+        use_dummy_transformation = options["use_dummy_transformation"]
+
+        if use_dummy_transformation:
+            transformation = DummyTransformation()
+        else:
+            transformation = JsonPersonsMemberships2OpdmTransformation()
 
         self.logger.info("Extraction from JSON")
         ETL(
             extractor=JsonArrayExtractor(source_url),
             loader=PopoloPersonWithMembershipsLoader(
-                context="parlamento_it",
+                context=context,
                 update_strategy=persons_update_strategy,
                 memberships_update_strategy=memberships_update_strategy,
                 check_membership_label=check_membership_label,
-                log_step=10,
+                log_step=log_step
             ),
-            transformation=JsonPersonsMemberships2OpdmTransformation(),
+            transformation=transformation,
             log_level=self.logger.level,
         )()
         self.logger.info("End")

project/api_v1/management/commands/script_correct_classifications.py

@@ -103,9 +103,13 @@ class Command(LoggingBaseCommand):
                 self.logger.error("Could not find classification for {0}".format(row["classification"]))
                 continue
             o_id = int(row["id"].split("/")[-1])
-            o = Organization.objects.get(id=o_id)
-            o.add_classification_rel(c)
-            o.classification = c.descr
-            o.save()
+            try:
+                o = Organization.objects.get(id=o_id)
+                o.add_classification_rel(c)
+                o.classification = c.descr
+                o.save()
+            except Organization.DoesNotExist:
+                self.logger.error("Could not find organization for {0}".format(o_id))
+                continue
 
         self.logger.info("End")

setup.cfg

 [bumpversion]
-current_version = 1.0.1
+current_version = 1.0.2
 commit = True
 tag = True
 tag_name = v{new_version}