projectal.entity

   1"""
   2The base Entity class that all entities inherit from.
   3"""
   4
   5import copy
   6import logging
   7import sys
   8
   9import projectal
  10from projectal import api
  11
  12
  13class Entity(dict):
  14    """
  15    The parent class for all our entities, offering requests
  16    and validation for the fundamental create/read/update/delete
  17    operations.
  18
  19    This class (and all our entities) inherit from the builtin
  20    `dict` class. This means all entity classes can be used
  21    like standard Python dictionary objects, but we can also
  22    offer additional utility functions that operate on the
  23    instance itself (see `linkers` for an example). Any method
  24    that expects a `dict` can also consume an `Entity` subclass.
  25
  26    The class methods in this class can operate on one or more
  27    entities in one request. If the methods are called with
  28    lists (for batch operation), the output returned will also
  29    be a list. Otherwise, a single `Entity` subclass is returned.
  30
  31    Note for batch operations: a `ProjectalException` is raised
  32    if *any* of the entities fail during the operation. The
  33    changes will *still be saved to the database for the entities
  34    that did not fail*.
  35    """
  36
  37    #: Child classes must override these with their entity names
  38    _path = "entity"  # URL portion to api
  39    _name = "entity"
  40
  41    # And to which entities they link to
  42    _links = []
  43    _links_reverse = []
  44
  45    def __init__(self, data):
  46        dict.__init__(self, data)
  47        self._is_new = True
  48        self._link_def_by_key = {}
  49        self._link_def_by_name = {}
  50        self._create_link_defs()
  51        self._with_links = set()
  52
  53        self.__fetch = self.get
  54        self.get = self.__get
  55        self.update = self.__update
  56        self.delete = self.__delete
  57        self.history = self.__history
  58        self.__type_links()
  59        self.__old = copy.deepcopy(self)
  60
  61    # ----- LINKING -----
  62
  63    def _create_link_defs(self):
  64        for cls in self._links:
  65            self._add_link_def(cls)
  66        for cls in self._links_reverse:
  67            self._add_link_def(cls, reverse=True)
  68
  69    def _add_link_def(self, cls, reverse=False):
  70        """
  71        Each entity is accompanied by a dict with details about how to
  72        get access to the data of the link within the object. Subclasses
  73        can pass in customizations to this dict when their APIs differ.
  74
  75        reverse denotes a reverse linker, where extra work is done to
  76        reverse the relationship of the link internally so that it works.
  77        The backend only offers one side of the relationship.
  78        """
  79        d = {
  80            "name": cls._link_name,
  81            "link_key": cls._link_key or cls._link_name + "List",
  82            "data_name": cls._link_data_name,
  83            "type": cls._link_type,
  84            "entity": cls._link_entity or cls._link_name.capitalize(),
  85            "reverse": reverse,
  86        }
  87        self._link_def_by_key[d["link_key"]] = d
  88        self._link_def_by_name[d["name"]] = d
  89
  90    def _add_link(self, to_entity_name, to_link):
  91        self._link(to_entity_name, to_link, "add", batch_linking=False)
  92
  93    def _update_link(self, to_entity_name, to_link):
  94        self._link(to_entity_name, to_link, "update", batch_linking=False)
  95
  96    def _delete_link(self, to_entity_name, to_link):
  97        self._link(to_entity_name, to_link, "delete", batch_linking=False)
  98
  99    def _link(
 100        self, to_entity_name, to_link, operation, update_cache=True, batch_linking=True
 101    ):
 102        """
 103        `to_entity_name`: Destination entity name (e.g. 'staff')
 104
 105        `to_link`: List of Entities of the same type (and optional data) to link to
 106
 107        `operation`: `add`, `update`, `delete`
 108
 109        'update_cache': also modify the entity's internal representation of the links
 110        to match the operation that was done. Set this to False when replacing the
 111        list with a new one (i.e., when calling save() instead of a linker method).
 112
 113        'batch_linking': Enabled by default, batches any link
 114        updates required into composite API requests. If disabled
 115        a request will be executed for each link update.
 116        Recommended to leave enabled to increase performance.
 117        """
 118
 119        link_def = self._link_def_by_name[to_entity_name]
 120        to_key = link_def["link_key"]
 121
 122        if isinstance(to_link, dict) and link_def["type"] == list:
 123            # Convert input dict to list when link type is a list (we allow linking to single entity for convenience)
 124            to_link = [to_link]
 125
 126            # For cases where user passed in dict instead of Entity, we turn them into
 127            # Entity on their behalf.
 128            typed_list = []
 129            target_cls = getattr(sys.modules["projectal.entities"], link_def["entity"])
 130            for link in to_link:
 131                if not isinstance(link, target_cls):
 132                    typed_list.append(target_cls(link))
 133                else:
 134                    typed_list.append(link)
 135            to_link = typed_list
 136        else:
 137            # For everything else, we expect types to match.
 138            if not isinstance(to_link, link_def["type"]):
 139                raise api.UsageException(
 140                    "Expected link type to be {}. Got {}.".format(
 141                        link_def["type"], type(to_link)
 142                    )
 143                )
 144
 145        if not to_link:
 146            return
 147
 148        url = ""
 149        payload = {}
 150        request_list = []
 151        # Is it a reverse linker? If so, invert the relationship
 152        if link_def["reverse"]:
 153            for link in to_link:
 154                request_list.extend(
 155                    link._link(
 156                        self._name,
 157                        self,
 158                        operation,
 159                        update_cache,
 160                        batch_linking=batch_linking,
 161                    )
 162                )
 163        else:
 164            # Only keep UUID and the data attribute, if it has one
 165            def strip_payload(link):
 166                single = {"uuId": link["uuId"]}
 167                data_name = link_def.get("data_name")
 168                if data_name and data_name in link:
 169                    single[data_name] = copy.deepcopy(link[data_name])
 170                return single
 171
 172            # If batch linking is enabled and the entity to link is a list of entities,
 173            # a separate request must be constructed for each one because the final composite
 174            # request permits only one input per call
 175            url = "/api/{}/link/{}/{}".format(self._path, to_entity_name, operation)
 176            to_link_payload = None
 177            if isinstance(to_link, list):
 178                to_link_payload = []
 179                for link in to_link:
 180                    if batch_linking:
 181                        request_list.append(
 182                            {
 183                                "method": "POST",
 184                                "invoke": url,
 185                                "body": {
 186                                    "uuId": self["uuId"],
 187                                    to_key: [strip_payload(link)],
 188                                },
 189                            }
 190                        )
 191                    else:
 192                        to_link_payload.append(strip_payload(link))
 193            if isinstance(to_link, dict):
 194                if batch_linking:
 195                    request_list.append(
 196                        {
 197                            "method": "POST",
 198                            "invoke": url,
 199                            "body": {
 200                                "uuId": self["uuId"],
 201                                to_key: strip_payload(to_link),
 202                            },
 203                        }
 204                    )
 205                else:
 206                    to_link_payload = strip_payload(to_link)
 207
 208            if not batch_linking:
 209                payload = {"uuId": self["uuId"], to_key: to_link_payload}
 210                api.post(url, payload=payload)
 211
 212        if not update_cache:
 213            return request_list
 214
 215        # Set the initial state if first add. We need the type to be set to correctly update the cache
 216        if operation == "add" and self.get(to_key, None) is None:
 217            if link_def.get("type") == dict:
 218                self[to_key] = {}
 219            elif link_def.get("type") == list:
 220                self[to_key] = []
 221
 222        # Modify the entity object's cache of links to match the changes we pushed to the server.
 223        if isinstance(self.get(to_key, []), list):
 224            if operation == "add":
 225                # Sometimes the backend doesn't return a list when it has none. Create it.
 226                if to_key not in self:
 227                    self[to_key] = []
 228
 229                for to_entity in to_link:
 230                    self[to_key].append(to_entity)
 231            else:
 232                for to_entity in to_link:
 233                    # Find it in original list
 234                    for i, old in enumerate(self.get(to_key, [])):
 235                        if old["uuId"] == to_entity["uuId"]:
 236                            if operation == "update":
 237                                self[to_key][i] = to_entity
 238                            elif operation == "delete":
 239                                del self[to_key][i]
 240        if isinstance(self.get(to_key, None), dict):
 241            if operation in ["add", "update"]:
 242                self[to_key] = to_link
 243            elif operation == "delete":
 244                self[to_key] = None
 245
 246        # Update the "old" record of the link on the entity to avoid
 247        # flagging it for changes (link lists are not meant to be user editable).
 248        if to_key in self:
 249            self.__old[to_key] = self[to_key]
 250
 251        return request_list
 252
 253    # -----
 254
 255    @classmethod
 256    def create(
 257        cls,
 258        entities,
 259        params=None,
 260        batch_linking=True,
 261        disable_system_features=True,
 262        enable_system_features_on_exit=True,
 263    ):
 264        """
 265        Create one or more entities of the same type. The entity
 266        type is determined by the subclass calling this method.
 267
 268        `entities`: Can be a `dict` to create a single entity,
 269        or a list of `dict`s to create many entities in bulk.
 270
 271        `params`: Optional URL parameters that may apply to the
 272        entity's API (e.g: `?holder=1234`).
 273
 274        'batch_linking': Enabled by default, batches any link
 275        updates required into composite API requests. If disabled
 276        a request will be executed for each link update.
 277        Recommended to leave enabled to increase performance.
 278
 279        If input was a `dict`, returns an entity subclass. If input was
 280        a list of `dict`s, returns a list of entity subclasses.
 281
 282        ```
 283        # Example usage:
 284        projectal.Customer.create({'name': 'NewCustomer'})
 285        # returns Customer object
 286        ```
 287        """
 288
 289        if isinstance(entities, dict):
 290            # Dict input needs to be a list
 291            e_list = [entities]
 292        else:
 293            # We have a list of dicts already, the expected format
 294            e_list = entities
 295
 296        # Apply type
 297        typed_list = []
 298        for e in e_list:
 299            if not isinstance(e, Entity):
 300                # Start empty to correctly populate history
 301                new = cls({})
 302                new.update(e)
 303                typed_list.append(new)
 304            else:
 305                typed_list.append(e)
 306        e_list = typed_list
 307
 308        endpoint = "/api/{}/add".format(cls._path)
 309        if params:
 310            endpoint += params
 311        if not e_list:
 312            return []
 313
 314        # Strip links from payload
 315        payload = []
 316        keys = e_list[0]._link_def_by_key.keys()
 317        for e in e_list:
 318            cleancopy = copy.deepcopy(e)
 319            # Remove any fields that match a link key
 320            for key in keys:
 321                cleancopy.pop(key, None)
 322            payload.append(cleancopy)
 323
 324        objects = []
 325        for i in range(0, len(payload), projectal.chunk_size_write):
 326            chunk = payload[i : i + projectal.chunk_size_write]
 327            orig_chunk = e_list[i : i + projectal.chunk_size_write]
 328            response = api.post(endpoint, chunk)
 329            # Put uuId from response into each input dict
 330            for e, o, orig in zip(chunk, response, orig_chunk):
 331                orig["uuId"] = o["uuId"]
 332                orig.__old = copy.deepcopy(orig)
 333                # Delete links from the history in order to trigger a change on them after
 334                for key in orig._link_def_by_key:
 335                    orig.__old.pop(key, None)
 336                objects.append(orig)
 337
 338        # Detect and apply any link additions
 339        # if batch_linking is enabled, builds a list of link requests
 340        # needed for each entity, then executes them with composite
 341        # API requests
 342        link_request_batch = []
 343        for e in e_list:
 344            requests = e.__apply_link_changes(batch_linking=batch_linking)
 345            link_request_batch.extend(requests)
 346
 347        if len(link_request_batch) > 0 and batch_linking:
 348            for i in range(0, len(link_request_batch), 100):
 349                chunk = link_request_batch[i : i + 100]
 350                if disable_system_features:
 351                    chunk = [
 352                        {
 353                            "note": "Disable Scheduling",
 354                            "invoke": "PUT /api/system/features?entity=scheduling&action=DISABLE",
 355                        },
 356                        {
 357                            "note": "Disable Macros",
 358                            "invoke": "PUT /api/system/features?entity=macros&action=disable",
 359                        },
 360                    ] + chunk
 361                if not enable_system_features_on_exit:
 362                    chunk.append(
 363                        {
 364                            "note": "Exit script execution, and do not restore some disabled commands",
 365                            "invoke": "PUT /api/system/features?entity=script&action=EXIT",
 366                        }
 367                    )
 368                api.post("/api/composite", chunk)
 369
 370        if not isinstance(entities, list):
 371            return objects[0]
 372        return objects
 373
 374    @classmethod
 375    def _get_linkset(cls, links):
 376        """Get a set of link names we have been asked to fetch with. Raise an
 377        error if the requested link is not valid for this Entity type."""
 378        link_set = set()
 379        if links is not None:
 380            if isinstance(links, str) or not hasattr(links, "__iter__"):
 381                raise projectal.UsageException(
 382                    "Parameter 'links' must be a list or None."
 383                )
 384
 385            defs = cls({})._link_def_by_name
 386            for link in links:
 387                name = link.lower()
 388                if name not in defs:
 389                    raise projectal.UsageException(
 390                        "Link '{}' is invalid for {}".format(name, cls._name)
 391                    )
 392                link_set.add(name)
 393        return link_set
 394
 395    @classmethod
 396    def get(cls, entities, links=None, deleted_at=None):
 397        """
 398        Get one or more entities of the same type. The entity
 399        type is determined by the subclass calling this method.
 400
 401        `entities`: One of several formats containing the `uuId`s
 402        of the entities you want to get (see bottom for examples):
 403
 404        - `str` or list of `str`
 405        - `dict` or list of `dict` (with `uuId` key)
 406
 407        `links`: A case-insensitive list of entity names to fetch with
 408        this entity. For performance reasons, links are only returned
 409        on demand.
 410
 411        Links follow a common naming convention in the output with
 412        a *_List* suffix. E.g.:
 413        `links=['company', 'location']` will appear as `companyList` and
 414        `locationList` in the response.
 415        ```
 416        # Example usage:
 417        # str
 418        projectal.Project.get('1b21e445-f29a-4a9f-95ff-fe253a3e1b11')
 419
 420        # list of str
 421        ids = ['1b21e445-f29a...', '1b21e445-f29a...', '1b21e445-f29a...']
 422        projectal.Project.get(ids)
 423
 424        # dict
 425        project = project.Project.create({'name': 'MyProject'})
 426        # project = {'uuId': '1b21e445-f29a...', 'name': 'MyProject', ...}
 427        projectal.Project.get(project)
 428
 429        # list of dicts (e.g. from a query)
 430        # projects = [{'uuId': '1b21e445-f29a...'}, {'uuId': '1b21e445-f29a...'}, ...]
 431        project.Project.get(projects)
 432
 433        # str with links
 434        projectal.Project.get('1b21e445-f29a...', 'links=['company', 'location']')
 435        ```
 436
 437        `deleted_at`: Include this parameter to get a deleted entity.
 438        This value should be a UTC timestamp from a webhook delete event.
 439        """
 440        link_set = cls._get_linkset(links)
 441
 442        if isinstance(entities, str):
 443            # String input is a uuId
 444            payload = [{"uuId": entities}]
 445        elif isinstance(entities, dict):
 446            # Dict input needs to be a list
 447            payload = [entities]
 448        elif isinstance(entities, list):
 449            # List input can be a list of uuIds or list of dicts
 450            # If uuIds (strings), convert to list of dicts
 451            if len(entities) > 0 and isinstance(entities[0], str):
 452                payload = [{"uuId": uuId} for uuId in entities]
 453            else:
 454                # Already expected format
 455                payload = entities
 456        else:
 457            # We have a list of dicts already, the expected format
 458            payload = entities
 459
 460        if deleted_at:
 461            if not isinstance(deleted_at, int):
 462                raise projectal.UsageException("deleted_at must be a number")
 463
 464        url = "/api/{}/get".format(cls._path)
 465        params = []
 466        params.append("links={}".format(",".join(links))) if links else None
 467        params.append("epoch={}".format(deleted_at - 1)) if deleted_at else None
 468        if len(params) > 0:
 469            url += "?" + "&".join(params)
 470
 471        # We only need to send over the uuIds
 472        payload = [{"uuId": e["uuId"]} for e in payload]
 473        if not payload:
 474            return []
 475        objects = []
 476        for i in range(0, len(payload), projectal.chunk_size_read):
 477            chunk = payload[i : i + projectal.chunk_size_read]
 478            dicts = api.post(url, chunk)
 479            for d in dicts:
 480                obj = cls(d)
 481                obj._with_links.update(link_set)
 482                obj._is_new = False
 483                # Create default fields for links we ask for. Workaround for backend
 484                # sometimes omitting links if no links exist.
 485                for link_name in link_set:
 486                    link_def = obj._link_def_by_name[link_name]
 487                    if link_def["link_key"] not in obj:
 488                        if link_def["type"] == dict:
 489                            obj.set_readonly(link_def["link_key"], None)
 490                        else:
 491                            obj.set_readonly(link_def["link_key"], link_def["type"]())
 492                objects.append(obj)
 493
 494        if not isinstance(entities, list):
 495            return objects[0]
 496        return objects
 497
 498    def __get(self, *args, **kwargs):
 499        """Use the dict get for instances."""
 500        return super(Entity, self).get(*args, **kwargs)
 501
 502    @classmethod
 503    def update(
 504        cls,
 505        entities,
 506        batch_linking=True,
 507        disable_system_features=True,
 508        enable_system_features_on_exit=True,
 509    ):
 510        """
 511        Save one or more entities of the same type. The entity
 512        type is determined by the subclass calling this method.
 513        Only the fields that have been modifier will be sent
 514        to the server as part of the request.
 515
 516        `entities`: Can be a `dict` to update a single entity,
 517        or a list of `dict`s to update many entities in bulk.
 518
 519        'batch_linking': Enabled by default, batches any link
 520        updates required into composite API requests. If disabled
 521        a request will be executed for each link update.
 522        Recommended to leave enabled to increase performance.
 523
 524        Returns `True` if all entities update successfully.
 525
 526        ```
 527        # Example usage:
 528        rebate = projectal.Rebate.create({'name': 'Rebate2022', 'rebate': 0.2})
 529        rebate['name'] = 'Rebate2024'
 530        projectal.Rebate.update(rebate)
 531        # Returns True. New rebate name has been saved.
 532        ```
 533        """
 534        if isinstance(entities, dict):
 535            e_list = [entities]
 536        else:
 537            e_list = entities
 538
 539        # allows for filtering of link keys
 540        typed_list = []
 541        for e in e_list:
 542            if not isinstance(e, Entity):
 543                new = cls({})
 544                new.update(e)
 545                typed_list.append(new)
 546            else:
 547                typed_list.append(e)
 548        e_list = typed_list
 549
 550        # Reduce the list to only modified entities and their modified fields.
 551        # Only do this to an Entity subclass - the consumer may have passed
 552        # in a dict of changes on their own.
 553        payload = []
 554
 555        for e in e_list:
 556            if isinstance(e, Entity):
 557                changes = e._changes_internal()
 558                if changes:
 559                    changes["uuId"] = e["uuId"]
 560                    payload.append(changes)
 561            else:
 562                payload.append(e)
 563        if payload:
 564            for i in range(0, len(payload), projectal.chunk_size_write):
 565                chunk = payload[i : i + projectal.chunk_size_write]
 566                api.put("/api/{}/update".format(cls._path), chunk)
 567
 568        # Detect and apply any link changes
 569        # if batch_linking is enabled, builds a list of link requests
 570        # from the changes of each entity, then executes
 571        # composite API requests with those changes
 572        link_request_batch = []
 573        for e in e_list:
 574            if isinstance(e, Entity):
 575                requests = e.__apply_link_changes(batch_linking=batch_linking)
 576                link_request_batch.extend(requests)
 577
 578        if len(link_request_batch) > 0 and batch_linking:
 579            for i in range(0, len(link_request_batch), 100):
 580                chunk = link_request_batch[i : i + 100]
 581                if disable_system_features:
 582                    chunk = [
 583                        {
 584                            "note": "Disable Scheduling",
 585                            "invoke": "PUT /api/system/features?entity=scheduling&action=DISABLE",
 586                        },
 587                        {
 588                            "note": "Disable Macros",
 589                            "invoke": "PUT /api/system/features?entity=macros&action=disable",
 590                        },
 591                    ] + chunk
 592                if not enable_system_features_on_exit:
 593                    chunk.append(
 594                        {
 595                            "note": "Exit script execution, and do not restore some disabled commands",
 596                            "invoke": "PUT /api/system/features?entity=script&action=EXIT",
 597                        }
 598                    )
 599                api.post("/api/composite", chunk)
 600
 601        return True
 602
 603    def __update(self, *args, **kwargs):
 604        """Use the dict update for instances."""
 605        return super(Entity, self).update(*args, **kwargs)
 606
 607    def save(self):
 608        """Calls `update()` on this instance of the entity, saving
 609        it to the database."""
 610        return self.__class__.update(self)
 611
 612    @classmethod
 613    def delete(cls, entities):
 614        """
 615        Delete one or more entities of the same type. The entity
 616        type is determined by the subclass calling this method.
 617
 618        `entities`: See `Entity.get()` for expected formats.
 619
 620        ```
 621        # Example usage:
 622        ids = ['1b21e445-f29a...', '1b21e445-f29a...', '1b21e445-f29a...']
 623        projectal.Customer.delete(ids)
 624        ```
 625        """
 626        if isinstance(entities, str):
 627            # String input is a uuId
 628            payload = [{"uuId": entities}]
 629        elif isinstance(entities, dict):
 630            # Dict input needs to be a list
 631            payload = [entities]
 632        elif isinstance(entities, list):
 633            # List input can be a list of uuIds or list of dicts
 634            # If uuIds (strings), convert to list of dicts
 635            if len(entities) > 0 and isinstance(entities[0], str):
 636                payload = [{"uuId": uuId} for uuId in entities]
 637            else:
 638                # Already expected format
 639                payload = entities
 640        else:
 641            # We have a list of dicts already, the expected format
 642            payload = entities
 643
 644        # We only need to send over the uuIds
 645        payload = [{"uuId": e["uuId"]} for e in payload]
 646        if not payload:
 647            return True
 648        for i in range(0, len(payload), projectal.chunk_size_write):
 649            chunk = payload[i : i + projectal.chunk_size_write]
 650            api.delete("/api/{}/delete".format(cls._path), chunk)
 651        return True
 652
 653    def __delete(self):
 654        """Let an instance delete itself."""
 655        return self.__class__.delete(self)
 656
 657    def clone(self, entity):
 658        """
 659        Clones an entity and returns its `uuId`.
 660
 661        Each entity has its own set of required values when cloning.
 662        Check the API documentation of that entity for details.
 663        """
 664        url = "/api/{}/clone?reference={}".format(self._path, self["uuId"])
 665        response = api.post(url, entity)
 666        return response["jobClue"]["uuId"]
 667
 668    @classmethod
 669    def history(cls, UUID, start=0, limit=-1, order="desc", epoch=None, event=None):
 670        """
 671        Returns an ordered list of all changes made to the entity.
 672
 673        `UUID`: the UUID of the entity.
 674
 675        `start`: Start index for pagination (default: `0`).
 676
 677        `limit`: Number of results to include for pagination. Use
 678        `-1` to return the entire history (default: `-1`).
 679
 680        `order`: `asc` or `desc` (default: `desc` (index 0 is newest))
 681
 682        `epoch`: only return the history UP TO epoch date
 683
 684        `event`:
 685        """
 686        url = "/api/{}/history?holder={}&".format(cls._path, UUID)
 687        params = []
 688        params.append("start={}".format(start))
 689        params.append("limit={}".format(limit))
 690        params.append("order={}".format(order))
 691        params.append("epoch={}".format(epoch)) if epoch else None
 692        params.append("event={}".format(event)) if event else None
 693        url += "&".join(params)
 694        return api.get(url)
 695
 696    def __history(self, **kwargs):
 697        """Get history of instance."""
 698        return self.__class__.history(self["uuId"], **kwargs)
 699
 700    @classmethod
 701    def list(cls, expand=False, links=None):
 702        """Return a list of all entity UUIDs of this type.
 703
 704        You may pass in `expand=True` to get full Entity objects
 705        instead, but be aware this may be very slow if you have
 706        thousands of objects.
 707
 708        If you are expanding the objects, you may further expand
 709        the results with `links`.
 710        """
 711
 712        payload = {
 713            "name": "List all entities of type {}".format(cls._name.upper()),
 714            "type": "msql",
 715            "start": 0,
 716            "limit": -1,
 717            "select": [["{}.uuId".format(cls._name.upper())]],
 718        }
 719        ids = api.query(payload)
 720        ids = [id[0] for id in ids]
 721        if ids:
 722            return cls.get(ids, links=links) if expand else ids
 723        return []
 724
 725    @classmethod
 726    def match(cls, field, term, links=None):
 727        """Find entities where `field`=`term` (exact match), optionally
 728        expanding the results with `links`.
 729
 730        Relies on `Entity.query()` with a pre-built set of rules.
 731        ```
 732        projects = projectal.Project.match('identifier', 'zmb-005')
 733        ```
 734        """
 735        filter = [["{}.{}".format(cls._name.upper(), field), "eq", term]]
 736        return cls.query(filter, links)
 737
 738    @classmethod
 739    def match_startswith(cls, field, term, links=None):
 740        """Find entities where `field` starts with the text `term`,
 741        optionally expanding the results with `links`.
 742
 743        Relies on `Entity.query()` with a pre-built set of rules.
 744        ```
 745        projects = projectal.Project.match_startswith('name', 'Zomb')
 746        ```
 747        """
 748        filter = [["{}.{}".format(cls._name.upper(), field), "prefix", term]]
 749        return cls.query(filter, links)
 750
 751    @classmethod
 752    def match_endswith(cls, field, term, links=None):
 753        """Find entities where `field` ends with the text `term`,
 754        optionally expanding the results with `links`.
 755
 756        Relies on `Entity.query()` with a pre-built set of rules.
 757        ```
 758        projects = projectal.Project.match_endswith('identifier', '-2023')
 759        ```
 760        """
 761        term = "(?i).*{}$".format(term)
 762        filter = [["{}.{}".format(cls._name.upper(), field), "regex", term]]
 763        return cls.query(filter, links)
 764
 765    @classmethod
 766    def match_one(cls, field, term, links=None):
 767        """Convenience function for match(). Returns the first match or None."""
 768        matches = cls.match(field, term, links)
 769        if matches:
 770            return matches[0]
 771
 772    @classmethod
 773    def match_startswith_one(cls, field, term, links=None):
 774        """Convenience function for match_startswith(). Returns the first match or None."""
 775        matches = cls.match_startswith(field, term, links)
 776        if matches:
 777            return matches[0]
 778
 779    @classmethod
 780    def match_endswith_one(cls, field, term, links=None):
 781        """Convenience function for match_endswith(). Returns the first match or None."""
 782        matches = cls.match_endswith(field, term, links)
 783        if matches:
 784            return matches[0]
 785
 786    @classmethod
 787    def search(cls, fields=None, term="", case_sensitive=True, links=None):
 788        """Find entities that contain the text `term` within `fields`.
 789        `fields` is a list of field names to target in the search.
 790
 791        `case_sensitive`: Optionally turn off case sensitivity in the search.
 792
 793        Relies on `Entity.query()` with a pre-built set of rules.
 794        ```
 795        projects = projectal.Project.search(['name', 'description'], 'zombie')
 796        ```
 797        """
 798        filter = []
 799        term = "(?{}).*{}.*".format("" if case_sensitive else "?", term)
 800        for field in fields:
 801            filter.append(["{}.{}".format(cls._name.upper(), field), "regex", term])
 802        filter = ["_or_", filter]
 803        return cls.query(filter, links)
 804
 805    @classmethod
 806    def query(cls, filter, links=None, timeout=30):
 807        """Run a query on this entity with the supplied filter.
 808
 809        The query is already set up to target this entity type, and the
 810        results will be converted into full objects when found, optionally
 811        expanded with the `links` provided. You only need to supply a
 812        filter to reduce the result set.
 813
 814        See [the filter documentation](https://projectal.com/docs/v1.1.1#section/Filter-section)
 815        for a detailed overview of the kinds of filters you can construct.
 816        """
 817        payload = {
 818            "name": "Python library entity query ({})".format(cls._name.upper()),
 819            "type": "msql",
 820            "start": 0,
 821            "limit": -1,
 822            "select": [["{}.uuId".format(cls._name.upper())]],
 823            "filter": filter,
 824            "timeout": timeout,
 825        }
 826        ids = api.query(payload)
 827        ids = [id[0] for id in ids]
 828        if ids:
 829            return cls.get(ids, links=links)
 830        return []
 831
 832    def profile_get(self, key):
 833        """Get the profile (metadata) stored for this entity at `key`."""
 834        return projectal.profile.get(key, self.__class__._name.lower(), self["uuId"])
 835
 836    def profile_set(self, key, data):
 837        """Set the profile (metadata) stored for this entity at `key`. The contents
 838        of `data` will completely overwrite the existing data dictionary."""
 839        return projectal.profile.set(
 840            key, self.__class__._name.lower(), self["uuId"], data
 841        )
 842
 843    def __type_links(self):
 844        """Find links and turn their dicts into typed objects matching their Entity type."""
 845
 846        for key, _def in self._link_def_by_key.items():
 847            if key in self:
 848                cls = getattr(projectal, _def["entity"])
 849                if _def["type"] == list:
 850                    as_obj = []
 851                    for link in self[key]:
 852                        as_obj.append(cls(link))
 853                elif _def["type"] == dict:
 854                    as_obj = cls(self[key])
 855                else:
 856                    raise projectal.UsageException("Unexpected link type")
 857                self[key] = as_obj
 858
 859    def changes(self):
 860        """Return a dict containing the fields that have changed since fetching the object.
 861        Dict values contain both the old and new values. E.g.: {'old': 'original', 'new': 'current'}.
 862
 863        In the case of link lists, there are three values: added, removed, updated. Only links with
 864        a data attribute can end up in the updated list, and the old/new dictionary is placed within
 865        that data attribute. E.g. for a staff-resource link:
 866        'updated': [{
 867            'uuId': '24eb4c31-0f92-49d1-8b4d-507ab939003e',
 868            'resourceLink': {'quantity': {'old': 2, 'new': 5}}
 869        }]
 870        """
 871        changed = {}
 872        for key in self.keys():
 873            link_def = self._link_def_by_key.get(key)
 874            if link_def:
 875                changes = self._changes_for_link_list(link_def, key)
 876                # Only add it if something in it changed
 877                for action in changes.values():
 878                    if len(action):
 879                        changed[key] = changes
 880                        break
 881            elif key not in self.__old and self[key] is not None:
 882                changed[key] = {"old": None, "new": self[key]}
 883            elif self.__old.get(key) != self[key]:
 884                changed[key] = {"old": self.__old.get(key), "new": self[key]}
 885        return changed
 886
 887    def _changes_for_link_list(self, link_def, key):
 888        changes = self.__apply_list(link_def, report_only=True)
 889        data_key = link_def["data_name"]
 890
 891        # For linked entities, we will only report their UUID, name (if it has one),
 892        # and the content of their data attribute (if it has one).
 893        def get_slim_list(entities):
 894            slim = []
 895            if isinstance(entities, dict):
 896                entities = [entities]
 897            for e in entities:
 898                fields = {"uuId": e["uuId"]}
 899                name = e.get("name")
 900                if name:
 901                    fields["name"] = e["name"]
 902                if data_key and e[data_key]:
 903                    fields[data_key] = e[data_key]
 904                slim.append(fields)
 905            return slim
 906
 907        out = {
 908            "added": get_slim_list(changes.get("add", [])),
 909            "updated": [],
 910            "removed": get_slim_list(changes.get("remove", [])),
 911        }
 912
 913        updated = changes.get("update", [])
 914        if updated:
 915            before_map = {}
 916            for entity in self.__old.get(key):
 917                before_map[entity["uuId"]] = entity
 918
 919            for entity in updated:
 920                old_data = before_map[entity["uuId"]][data_key]
 921                new_data = entity[data_key]
 922                diff = {}
 923                for key in new_data.keys():
 924                    if key not in old_data and new_data[key] is not None:
 925                        diff[key] = {"old": None, "new": new_data[key]}
 926                    elif old_data.get(key) != new_data[key]:
 927                        diff[key] = {"old": old_data.get(key), "new": new_data[key]}
 928                out["updated"].append({"uuId": entity["uuId"], data_key: diff})
 929        return out
 930
 931    def _changes_internal(self):
 932        """Return a dict containing only the fields that have changed and their current value,
 933        without any link data.
 934
 935        This method is used internally to strip payloads down to only the fields that have changed.
 936        """
 937        changed = {}
 938        for key in self.keys():
 939            # We don't deal with link or link data changes here. We only want standard fields.
 940            if key in self._link_def_by_key:
 941                continue
 942            if key not in self.__old and self[key] is not None:
 943                changed[key] = self[key]
 944            elif self.__old.get(key) != self[key]:
 945                changed[key] = self[key]
 946        return changed
 947
 948    def set_readonly(self, key, value):
 949        """Set a field on this Entity that will not be sent over to the
 950        server on update unless modified."""
 951        self[key] = value
 952        self.__old[key] = value
 953
 954    # --- Link management ---
 955
 956    @staticmethod
 957    def __link_data_differs(have_link, want_link, data_key):
 958        if data_key:
 959            if "uuId" in have_link[data_key]:
 960                del have_link[data_key]["uuId"]
 961            if "uuId" in want_link[data_key]:
 962                del want_link[data_key]["uuId"]
 963            return have_link[data_key] != want_link[data_key]
 964
 965        # Links without data never differ
 966        return False
 967
 968    def __apply_link_changes(self, batch_linking=True):
 969        """Send each link list to the conflict resolver. If we detect
 970        that the entity was not fetched with that link, we do the fetch
 971        first and use the result as the basis for comparison."""
 972
 973        # Find which lists belong to links but were not fetched so we can fetch them
 974        need = []
 975        find_list = []
 976        if not self._is_new:
 977            for link in self._link_def_by_key.values():
 978                if link["link_key"] in self and link["name"] not in self._with_links:
 979                    need.append(link["name"])
 980                    find_list.append(link["link_key"])
 981
 982        if len(need):
 983            logging.warning(
 984                "Entity links were modified but entity not fetched with links. "
 985                "For better performance, include the links when getting the entity."
 986            )
 987            logging.warning(
 988                "Fetching {} again with missing links: {}".format(
 989                    self._name.upper(), ",".join(need)
 990                )
 991            )
 992            new = self.__fetch(self, links=need)
 993            for _list in find_list:
 994                self.__old[_list] = copy.deepcopy(new.get(_list, []))
 995
 996        # if batch_linking is enabled, builds a list of link requests
 997        # for each link definition of the calling entity then returns the list
 998        request_list = []
 999        for link_def in self._link_def_by_key.values():
1000            link_def_requests = self.__apply_list(link_def, batch_linking=batch_linking)
1001            if batch_linking:
1002                request_list.extend(link_def_requests)
1003        return request_list
1004
1005    def __apply_list(self, link_def, report_only=False, batch_linking=True):
1006        """Automatically resolve differences and issue the correct sequence of
1007        link/unlink/relink for the link list to result in the supplied list
1008        of entities.
1009
1010        report_only will not make any changes to the data or issue network requests.
1011        Instead, it returns the three lists of changes (add, update, delete).
1012        """
1013        to_add = []
1014        to_remove = []
1015        to_update = []
1016        should_only_have = set()
1017        link_key = link_def["link_key"]
1018
1019        if link_def["type"] == list:
1020            want_entities = self.get(link_key, [])
1021            have_entities = self.__old.get(link_key, [])
1022
1023            if not isinstance(want_entities, list):
1024                raise api.UsageException(
1025                    "Expecting '{}' to be {}. Found {} instead.".format(
1026                        link_key,
1027                        link_def["type"].__name__,
1028                        type(want_entities).__name__,
1029                    )
1030                )
1031
1032            for want_entity in want_entities:
1033                if want_entity["uuId"] in should_only_have:
1034                    raise api.UsageException(
1035                        "Duplicate {} in {}".format(link_def["name"], link_key)
1036                    )
1037                should_only_have.add(want_entity["uuId"])
1038                have = False
1039                for have_entity in have_entities:
1040                    if have_entity["uuId"] == want_entity["uuId"]:
1041                        have = True
1042                        data_name = link_def.get("data_name")
1043                        if data_name and self.__link_data_differs(
1044                            have_entity, want_entity, data_name
1045                        ):
1046                            to_update.append(want_entity)
1047                if not have:
1048                    to_add.append(want_entity)
1049            for have_entity in have_entities:
1050                if have_entity["uuId"] not in should_only_have:
1051                    to_remove.append(have_entity)
1052        elif link_def["type"] == dict:
1053            # Note: dict type does not implement updates as we have no dict links
1054            # that support update (yet?).
1055            want_entity = self.get(link_key, None)
1056            have_entity = self.__old.get(link_key, None)
1057
1058            if want_entity is not None and not isinstance(want_entity, dict):
1059                raise api.UsageException(
1060                    "Expecting '{}' to be {}. Found {} instead.".format(
1061                        link_key, link_def["type"].__name__, type(have_entity).__name__
1062                    )
1063                )
1064
1065            if want_entity:
1066                if have_entity:
1067                    if want_entity["uuId"] != have_entity["uuId"]:
1068                        to_remove = have_entity
1069                        to_add = want_entity
1070                else:
1071                    to_add = want_entity
1072            if not want_entity:
1073                if have_entity:
1074                    to_remove = have_entity
1075
1076            want_entities = want_entity
1077        else:
1078            # Would be an error in this library if we reach here
1079            raise projectal.UnsupportedException("This type does not support linking")
1080
1081        # if batch_linking is enabled, builds a list of requests
1082        # from each link method
1083        if not report_only:
1084            request_list = []
1085            if to_remove:
1086                delete_requests = self._link(
1087                    link_def["name"],
1088                    to_remove,
1089                    "delete",
1090                    update_cache=False,
1091                    batch_linking=batch_linking,
1092                )
1093                request_list.extend(delete_requests)
1094            if to_update:
1095                update_requests = self._link(
1096                    link_def["name"],
1097                    to_update,
1098                    "update",
1099                    update_cache=False,
1100                    batch_linking=batch_linking,
1101                )
1102                request_list.extend(update_requests)
1103            if to_add:
1104                add_requests = self._link(
1105                    link_def["name"],
1106                    to_add,
1107                    "add",
1108                    update_cache=False,
1109                    batch_linking=batch_linking,
1110                )
1111                request_list.extend(add_requests)
1112            self.__old[link_key] = copy.deepcopy(want_entities)
1113            return request_list
1114        else:
1115            changes = {}
1116            if to_remove:
1117                changes["remove"] = to_remove
1118            if to_update:
1119                changes["update"] = to_update
1120            if to_add:
1121                changes["add"] = to_add
1122            return changes
1123
1124    @classmethod
1125    def get_link_definitions(cls):
1126        return cls({})._link_def_by_name
1127
1128    # --- ---
1129
1130    def entity_name(self):
1131        return self._name.capitalize()