.bumpversion.cfg

 [bumpversion]
-current_version = 0.3.1
+current_version = 0.3.2
 commit = True
 
 [bumpversion:file:web/opdm_service/__init__.py]

.gitignore

@@ -44,4 +44,4 @@ node_modules/
 dump.rdb
 dump.sql
 
-
+.ipynb_checkpoints

CHANGELOG.md

@@ -5,9 +5,23 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- Multiple overlapping memberships are possible if the `allow_overlap`
+  flag is specified
+- When memberships refers to overlapping dates a warning is emitted
+
+## [0.3.1]
+### Fixed
+
+- labels for Posts are now separated from the extended descriptions
+  while importing charges from Openpolis
+
+
+## [0.3.0]
 ### Added
 - import for persons from openpolis API started
 
+
 ## [0.2.0]
 ### Added
 - REST API endpoints for areas and organizations
@@ -34,6 +48,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 libspatialite5 added to Docker image; popolo and etml tests invoked
 in the test stage.
 
+
 ## [0.1.0]
 ### Added
 - first release: areas and organizations import management tasks added

web/opdm_service/__init__.py

 "The Openpolis Data Manager service package (the backend)"
 
-__version__ = '0.3.1'
+__version__ = '0.3.2'

web/opdm_service/etl/op_api.py

@@ -157,9 +157,11 @@ class OpAPIImporter(object):
                 m = state_re.match(op_pol.birth_location)
                 if m is None:
                     self.logger.warning(
-                        "Birth location parsing for CF computation "
-                        "was not possible :{0}:."
-                        "Skipping.".format(birth_location))
+                        "Parsing birth location :{0}: for CF computation "
+                        "for {1} was not possible.".format(
+                            birth_location, op_id
+                        )
+                    )
                 else:
                     birth_place['state'] = m.groupdict()['state']
             else:
@@ -179,7 +181,13 @@ class OpAPIImporter(object):
                     birth_place['city'],
                 )
             except Exception as e:
-                self.logger.warning(e)
+                self.logger.warning(
+                    "CF computation "
+                    "for {0} was not possible. "
+                    "Exception {1} detected.".format(
+                        op_id, e
+                    )
+                )
 
             # the birth_location_area Area object is determined
             # only for italian birth locations,
@@ -196,7 +204,9 @@ class OpAPIImporter(object):
                 except Exception as e:
                     self.logger.warning(
                         "Could not link birth location area."
-                        "{0}".format(bp_tuple)
+                        "{0} while processing {1}".format(
+                            bp_tuple, op_id
+                        )
                     )
 
         # other fields are computed and stored into defaults
@@ -251,7 +261,7 @@ class OpAPIImporter(object):
             defaults=person_defaults
         )
         if created:
-            self.logger.info(
+            self.logger.debug(
                 "Person created: {0} (op_id: {1})".format(
                     name, identifier.id
                 )
@@ -301,16 +311,47 @@ class OpAPIImporter(object):
         # return the imported Person instance
         return person
 
-    def import_memberships(self, inst_name, op_institution_id, post_label):
+    def import_memberships(self, inst_name, op_institution_id, build_labels):
         """Import all charges for a given institution.
 
         The institution is identified on Openpolis API by its ID
         It is given the specified `inst_name` name.
 
-        The memberships are all bound to generic Posts, whose label is
-        built using the post_label function.
+        The memberships are all bound to generic Posts.
+
+        Membership's and Post's labels are built through the
+        ``build_labels`` callable.
+
+        The following fields may be given values within the
+        `build_labels` callable.
+
+        Membership
+        - `label`: a label describing the membership
+        - `role`: the role that the member fullfils in the organization.
 
-        :param post_label: a callable to build the Post's label
+        Post
+        - `label`: a label describing the role
+        - `other_label`: an alternate label, such as an abbreviation
+        - `role`: the function that the holder of the post fulfills.
+
+        Note: The role property appears on both the Post and Membership classes,
+        as there are uses cases where you will have no posts
+        (e.g. describing club membership)
+        and others where you will have no memberships
+        (e.g. describing organizational structure).
+
+
+        Values will be available in a dictionary, with the following structure:
+        - membership
+          - label
+          - role
+        - post
+          - label
+          - other_label
+          - role
+
+        :param build_labels: a callable to build the Post's and Membership's
+                             labels and roles strings
         :param op_institution_id:
         :param inst_name:
         :return:
@@ -323,7 +364,7 @@ class OpAPIImporter(object):
         if cr:
             self.logger.debug("Org {0} created.".format(inst_name))
 
-        url = "{0}?institution_id={1}".format(
+        url = "{0}?institution_id={1}&page_size=100".format(
             self.api_base_url,
             op_institution_id
         )
@@ -350,27 +391,44 @@ class OpAPIImporter(object):
                         op_politician,
                     )
 
+                    # retrieve labels for membership and post
+                    # from the callable, filling all missing keys
+                    labels = build_labels(org, op_post)
+                    if 'post' not in labels:
+                        labels['post'] = {}
+                    if 'membership' not in labels:
+                        labels['membership'] = {}
+
                     post, cr = Post.objects.get_or_create(
-                        label=post_label(org, op_post)
+                        label=labels['post'].get('label', None),
+                        role=labels['post'].get('role', None)
                     )
                     if cr:
-                        self.logger.info("Post created: {0}".format(post.label))
+                        self.logger.debug("Post created: {0}".format(post.label))
 
                     membership = person.add_role(
                         post, organization=org,
                         start_date=op_post.date_start,
                         end_date=op_post.date_end,
+                        label=labels['membership'].get('label', None),
+                        role=labels['membership'].get('role', None),
                         created_at=op_post.content.content.created_at+"+0100",
                         updated_at=op_post.content.content.updated_at+"+0100"
                     )
                     if membership:
-                        self.logger.info(
-                            "Membership created: {0} - {1} ({2} - {3})".format(
+                        self.logger.debug(
+                            "Membership created: {0}({1}) - {1} - {2}".format(
+                                person, op_politician.content.id,
+                                post,
+                                membership
+                            )
+                        )
+                    else:
+                        self.logger.warning(
+                            "Membership could NOT be created: PE:{0} - PO:{1} - O:{2}".format(
                                 person,
                                 post,
-                                membership.start_date,
-                                membership.end_date or '-',
-                                'CREATED'
+                                org
                             )
                         )
 

web/opdm_service/management/commands/import_posts_and_persons_from_op.py

@@ -39,25 +39,35 @@ class Command(BaseCommand):
 
         self.op_importer.import_memberships(
             'Commissione Europea', 1,
-            lambda org, op_post: "{0} della {1}".format(
-                op_post.charge_type_descr,
-                org.name
-            )
+            lambda org, op_post: {
+                'post': {
+                    'label': '{0} della {1}'.format(
+                        op_post.charge_type_descr,
+                        org.name
+                    )
+                },
+            }
         )
 
-        def build_euparl_label(org, op_post):
+        def build_labels_parl_eu(org, op_post):
+            labels = {'post': {}, 'membership': {}}
             if op_post.charge_type_descr == 'Deputato':
-                label = "Parlamentare europeo"
+                labels['post'] = {
+                    'label': 'Parlamentare europeo',
+                    'other_label': 'Parl. EU',
+                }
             else:
-                label = "{0} del {1}".format(
-                    op_post.charge_type_descr,
-                    org.name
-                )
-            return label
+                labels['post'] = {
+                     'label': '{0} del {1}'.format(
+                        op_post.charge_type_descr,
+                        org.name
+                    )
+                }
+            return labels
 
         self.op_importer.import_memberships(
             'Parlamento Europeo', 2,
-            build_euparl_label
+            build_labels_parl_eu
         )
 
         self.logger.info('End UE')
@@ -65,18 +75,22 @@ class Command(BaseCommand):
     def handle_it(self):
         self.logger.info('Start IT')
 
-        def build_gov_label(org, op_post):
-            if op_post.charge_type_descr == 'Ministro' and\
-                op_post.description is not None:
-                label = "{0} con deleghe a: {1}".format(
+        def build_labels_gov(org, op_post):
+            labels = {'post': {}, 'membership': {}}
+            labels['post'] = {
+                'label': op_post.charge_type_descr,
+            }
+            if op_post.charge_type_descr == 'Ministro':
+                labels['post']['other_label'] = 'Min.'
+
+            if op_post.description is not None and op_post.description != '':
+                labels['membership']['role'] = "{0} con deleghe {1}".format(
                     op_post.charge_type_descr,
                     op_post.description
                 )
-            else:
-                label = op_post.charge_type_descr
-            return label
+            return labels
         self.op_importer.import_memberships(
-            'Governo', 3, build_gov_label
+            'Governo', 3, build_labels_gov
         )
 
         self.logger.info('End IT')