Add create custom object

.gitignore

@@ -62,3 +62,6 @@ target/
 .idea/
 
 .DS_Store
+
+# test config
+tests/zdtestcfg.py

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.8.6
+- Add support for cursor based pagination and make it default
+- Allow disabling cursor based pagination, because certain endpoints don't support it yet.
+- Add support for incremental api cursor and time pagination.
+
 ## 2.8.0
 - Regenerate API from updated mirror. see [full
   commit](https://github.com/fprimex/zdesk/commit/4982b3dad9581fbb49d71307abc229dc4169ab74).

CONTRIBUTORS.md

@@ -9,6 +9,7 @@
 * Dominik Miedziński (Booksy International Sp. z o. o.)
 * Sarfaraz Soomro (Incremental ticket pagination)
 * Craig Davis (Major `api_gen` updates for zdesk 2.6.0)
+* Otto Tamas (Pagination improvements - zdesk 2.8.6)
 
 
 ## zendesk and zdesk 1.x

README.md

@@ -154,19 +154,24 @@ equal to (the integer) `201`.
 
 ## Getting all pages
 
-There is a common pattern where a request will return one page of data along
-with a `next_page` location. In order to retrieve all results, it is necessary
-to continue retrieving every `next_page` location. The results then all need to
-be processed together. A loop to get all pages ends up stamped throughout
-Zendesk code, since many API methods return paged lists of objects.
-
-As a convenience, passing `get_all_pages` to any API method will do this for
-you, and will also merge all responses. The result is a single, large object
-that appears to be the result of one single call. The logic for this
-combination and reduction is well documented in the
-[source](https://github.com/fprimex/zdesk/blob/master/zdesk/zdesk.py#L534)
-(look for the line reading `Now we need to try to combine or reduce the
-results`, if the line number has shifted since this writing).
+Some endpoints support pagination in order to efficiently serve a larger
+amount of data. As a convenience, passing `get_all_pages` to endpoints
+that support pagination will follow links to progressively fetch all
+data. The result is a single, large object that appears to be the result 
+of one single call.
+
+- by default, cursor based pagination is used as recommended by the
+zdesk API documentation to favor efficiency.
+- you can optionally opt for falling back to offset based pagination by
+passing `cursor_pagination=False` flag.
+- a limited number of endpoints don't support cursor based pagination yet.
+If you get errors saying that `page[size]` parameter is not supported, then
+try setting `cursor_pagination=False`. Note that offset based pagination now
+has a hard limit (usually 10k) on the returned number of items, so I discourage 
+using it for endpoints that support cursor based pagination.
+- the incremental API has different mechanics for pagination, which are also
+supported by this lib. `cursor_pagination=False` might be required for most
+incremental API endpoints.
 
 ## MIME types for data
 

setup.py

@@ -5,7 +5,7 @@
     # Basic package information.
     name = 'zdesk',
     author = 'Brent Woodruff',
-    version = '2.8.0',
+    version = '2.8.8',
     author_email = 'brent@fprimex.com',
     packages = ['zdesk'],
     include_package_data = True,

tests/paginate/test_cursor_pagination.py

@@ -0,0 +1,27 @@
+import pytest
+
+from zdesk import Zendesk
+
+@pytest.fixture(scope="module", autouse=True)
+def expected_ticket_count(zd: Zendesk):
+    response = zd.tickets_count_list()
+    count = response["count"]["value"]
+    assert count >= 100, "Please create at least 100 tickets for this test to run"
+    return count
+
+
+def test_ticket_cursor_pagination(zd: Zendesk, expected_ticket_count: int):
+    response = zd.tickets_list(get_all_pages=True)
+    tickets = response["tickets"]
+    assert len(tickets) == expected_ticket_count
+
+def test_ticket_cursor_pagination_custom_page_size(zd: Zendesk, expected_ticket_count: int):
+    response = zd.tickets_list(page_size=50, get_all_pages=True)
+    tickets = response["tickets"]
+    assert len(tickets) == expected_ticket_count
+
+
+def test_ticket_offset_pagination(zd: Zendesk, expected_ticket_count: int):
+    response = zd.tickets_list(per_page=100, get_all_pages=True, cursor_pagination=False)
+    tickets = response["tickets"]
+    assert len(tickets) == expected_ticket_count

tests/paginate/test_incremental_api_pagination.py

@@ -0,0 +1,23 @@
+import datetime
+
+from zdesk import Zendesk
+
+days_ago = 60
+start_time = (datetime.datetime.now() - datetime.timedelta(days=days_ago)).timestamp()
+incremental_api_page_size = 1000
+
+def test_incremental_ticket_pagination(zd: Zendesk):
+    response = zd.incremental_tickets_list(get_all_pages=True, cursor_pagination=False, start_time=start_time)
+    time_pagination_tickets = response["tickets"]
+    assert response["end_time"] > start_time
+    assert response["end_of_stream"] is True
+    assert len(time_pagination_tickets) > incremental_api_page_size
+
+    response = zd.incremental_tickets_cursor_list(get_all_pages=True, cursor_pagination=False, start_time=start_time)
+    cursor_pagination_tickets = response["tickets"]
+    assert response["end_of_stream"] is True
+
+    # time pagination provides some duplicates
+    time_pagination_ticket_ids = {ticket["id"] for ticket in time_pagination_tickets}
+    cursor_pagination_tickets_ids = {ticket["id"] for ticket in cursor_pagination_tickets}
+    assert time_pagination_ticket_ids == cursor_pagination_tickets_ids

tests/paginate/test_offset_pagination.py

@@ -0,0 +1,29 @@
+import pytest
+
+from zdesk import Zendesk, ZendeskError
+
+@pytest.fixture(scope="module", autouse=True)
+def expected_ticket_count(zd: Zendesk):
+    response = zd.search_count(query="*")
+    count = response["count"]
+    assert count >= 100, "Please create at least 100 tickets for this test to run"
+    return count
+
+
+def test_search_offset_pagination(zd: Zendesk, expected_ticket_count: int):
+    # Try to implement a way to return partial results.
+    # This includes refactoring, because results are merged together at the end
+    # of pagination.
+    try:
+        response = zd.search(query="*", get_all_pages=True, cursor_pagination=False)
+    except ZendeskError as e:
+        assert len(e.partial_results["results"]) == 1000
+        assert e.partial_results["count"] == expected_ticket_count
+    else:
+        assert len(response["results"]) == expected_ticket_count
+
+def test_search_with_cursor_pagination_is_unsupported(zd: Zendesk):
+    with pytest.raises(ZendeskError) as e:
+        zd.search(query="*", get_all_pages=True, cursor_pagination=True)
+
+    assert e.value.partial_results is None

zdesk/zdesk.py

@@ -72,10 +72,11 @@ def get_id_from_url(url):
 
 
 class ZendeskError(Exception):
-    def __init__(self, msg, code, response):
+    def __init__(self, msg, code, response, partial_results = None):
         self.msg = msg
         self.error_code = code
         self.response = response
+        self.partial_results = partial_results
 
     def __str__(self):
         return repr('%s: %s %s' % (self.error_code, self.msg, self.response))
@@ -309,7 +310,7 @@ def max_retries(self):
     def call(self, path, query=None, method='GET', data=None,
              files=None, get_all_pages=False, complete_response=False,
              retry_on=None, max_retries=0, raw_query=None, retval=None,
-             **kwargs):
+             cursor_pagination=True, **kwargs):
         """Make a REST call to the Zendesk web service.
 
         Parameters:
@@ -333,6 +334,10 @@ def call(self, path, query=None, method='GET', data=None,
             appended to the URL path and will completely override / discard
             any other query parameters. Enables use cases where query
             parameters need to be repeated in the query string.
+        cursor_pagination - Whether to use cursor-based pagination or not.
+            Defaults to True. Some endpoints don't support cursor-based
+            pagination yet, so you can set this to False to revert to
+            offset/page based pagination.
         retval - Request a specific part of the returned response. Valid
             values are 'content', 'code', 'location', and 'headers'.
             JSON content is still automatically deserialized if possible.
@@ -373,6 +378,9 @@ def call(self, path, query=None, method='GET', data=None,
             else:
                 kwargs = query
 
+        if get_all_pages and cursor_pagination:
+            kwargs['page[size]'] = kwargs.pop('per_page', 100)
+
         if raw_query:
             path = path + raw_query
             kwargs = None
@@ -434,47 +442,26 @@ def call(self, path, query=None, method='GET', data=None,
 
             code = response.status_code
             try:
-                if not 200 <= code < 300 and code != 422:
+                if not 200 <= code < 300:
+                    partial_results = self._combine_results(results) if results else None
                     if code == 401:
                         raise AuthenticationError(
-                            response.content, code, response)
+                            response.content, code, response, partial_results)
                     elif code == 429:
                         raise RateLimitError(
-                            response.content, code, response)
+                            response.content, code, response, partial_results)
                     else:
                         raise ZendeskError(
-                            response.content, code, response)
+                            response.content, code, response, partial_results)
             except ZendeskError:
-                if request_count <= self.max_retries:
+                if request_count <= self.max_retries and code != 422:
                     self._handle_retry(response)
                     continue
                 else:
                     raise
 
-            # Deserialize json content if content exists.
-            # In some cases Zendesk returns ' ' strings.
-            # Also return false non strings (0, [], (), {})
-            if response.content.strip() and 'json' in response.headers['content-type']:
-                content = response.json()
-
-                # set url to the next page if that was returned in the response
-                url = content.get('next_page', None)
-                # url we get above already has the start_time appended to it,
-                # specific to incremental exports
-                kwargs = {}
-            elif response.content.strip() and 'text' in response.headers['content-type']:
-                try:
-                    content = response.json()
-                    # set url to the next page if that was returned in the response
-                    url = content.get('next_page', None)
-                    # url we get above already has the start_time appended to it,
-                    # specific to incremental exports
-                    kwargs = {}
-                except ValueError:
-                    content = response.content
-            else:
-                content = response.content
-                url = None
+
+            content, url, kwargs = self._parse_response(response, kwargs)
 
             if complete_response:
                 results.append({
@@ -519,6 +506,7 @@ def call(self, path, query=None, method='GET', data=None,
             # also note that incremental/ticket_metric_events end-point has a 10,000 items per page limit
             url = None if (url is not None and
                            'incremental' in url and
+                           'count' in content and
                            content.get('count') < 1000) else url
             all_requests_complete = not (get_all_pages and url)
             request_count = 0
@@ -553,7 +541,47 @@ def call(self, path, query=None, method='GET', data=None,
             # we have a list of simple objects like strings, but they are not
             # all the same so send them all back.
             return results
+
+        return self._combine_results(results)
+
+    def _parse_response(self, response, kwargs):
+        # Deserialize json content if content exists.
+        # In some cases Zendesk returns ' ' strings.
+        # Also return false non strings (0, [], (), {})
+        if not response.content.strip():
+            return response.content, None, kwargs
+
+        if 'json' in response.headers['content-type']:
+            json = response.json()
+        elif 'text' in response.headers['content-type']:
+            try:
+                json = response.json()
+            except ValueError:
+                json = None
+
+        if json:
+            url = None
+
+            # set url to the next page if that was returned in the response
+            if {'meta', 'links'} <= json.keys():
+                # cursor based pagination
+                if json.get('meta', {}).get('has_more'):
+                    url = json.get('links', {}).get('next')
+            else:
+                # offset based pagination
+                if not json.get('end_of_stream', False):
+                    url = json.get('next_page', None)
+
+                    # incremental api cursor pagination uses after_url instead
+                    url = url if url else json.get('after_url', None)
+
+            # url we get above already has kwargs appended,
+            return json, url, {}
+
+        return response.content, None, kwargs
+
 
+    def _combine_results(self, results):
         # may have a sequence of response contents
         # (dicts, possibly lists in the future as that is valid json also)
         combined_dict_results = {}

zdesk/zdesk_api.py

@@ -2539,6 +2539,11 @@ def help_center_sections_list(self, locale=None, **kwargs):
             api_path = api_opt_path.format(locale=locale)
         return self.call(api_path, **kwargs)
 
+    def help_center_integration_keys(self, **kwargs):
+        "https://developer.zendesk.com/api-reference/help_center/help-center-api/help_center_jwts/#list-public-keys"
+        api_path = "/api/v2/help_center/integration/keys"
+        return self.call(api_path, **kwargs)
+
     def help_center_translation_delete(self, translation_id, **kwargs):
         "https://developer.zendesk.com/api-reference/help_center/help-center-api/translations#delete-translation"
         api_path = "/api/v2/help_center/translations/{translation_id}"
@@ -4586,6 +4591,19 @@ def tickets_update_many(self, data, **kwargs):
         api_path = "/api/v2/tickets/update_many"
         return self.call(api_path, method="PUT", data=data, **kwargs)
 
+    def tickets_update_many_by_fields_dict(self, data, ids=None, **kwargs):
+        "https://developer.zendesk.com/rest_api/docs/core/tickets#update-many-tickets"
+        api_path = "/api/v2/tickets/update_many.json"
+        api_query = {}
+        if "query" in kwargs.keys():
+            api_query.update(kwargs["query"])
+            del kwargs["query"]
+        if ids:
+            api_query.update({
+                "ids": ids,
+            })
+        return self.call(api_path, query=api_query, method="PUT", data=data, **kwargs)
+
     def trigger_categories_job_create(self, data, **kwargs):
         "https://developer.zendesk.com/api-reference/ticketing/trigger_categories#create-batch-job-for-trigger-categories"
         api_path = "/api/v2/trigger_categories/jobs"
@@ -5149,10 +5167,22 @@ def users_request_create(self, data, **kwargs):
         api_path = "/api/v2/users/request_create"
         return self.call(api_path, method="POST", data=data, **kwargs)
 
-    def users_search(self, **kwargs):
-        "https://developer.zendesk.com/api-reference/ticketing/users#search-users"
-        api_path = "/api/v2/users/search"
-        return self.call(api_path, **kwargs)
+    def users_search(self, external_id=None, query=None, **kwargs):
+        "https://developer.zendesk.com/rest_api/docs/core/users#search-users"
+        api_path = "/api/v2/users/search.json"
+        api_query = {}
+        if "query" in kwargs.keys():
+            api_query.update(kwargs["query"])
+            del kwargs["query"]
+        if external_id:
+            api_query.update({
+                "external_id": external_id,
+            })
+        if query:
+            api_query.update({
+                "query": query,
+            })
+        return self.call(api_path, query=api_query, **kwargs)
 
     def users_show_many(self, **kwargs):
         "https://developer.zendesk.com/api-reference/ticketing/users#show-many-users"
@@ -5373,4 +5403,21 @@ def workspaces_reorder(self, data, **kwargs):
         api_path = "/api/v2/workspaces/reorder"
         return self.call(api_path, method="PUT", data=data, **kwargs)
 
+    def custom_object_record_create(self, custom_object_key, data, **kwargs):
+        "https://developer.zendesk.com/api-reference/custom-data/custom-objects/custom_object_records/#create-custom-object-record"
+        api_path = "/api/v2/custom_objects/{custom_object_key}/records"
+        api_path = api_path.format(custom_object_key=custom_object_key)
+        return self.call(api_path, method="POST", data=data, **kwargs)
+
+    def custom_object_record_show(self, custom_object_key, custom_object_record_id, data, **kwargs):
+        "https://developer.zendesk.com/api-reference/custom-data/custom-objects/custom_object_records/#show-custom-object-record"
+        api_path = "/api/v2/custom_objects/{custom_object_key}/records/{custom_object_record_id}"
+        api_path = api_path.format(custom_object_key=custom_object_key, custom_object_record_id=custom_object_record_id)
+        return self.call(api_path, method="GET", data=data, **kwargs)
+
+    def custom_object_record_delete(self, custom_object_key, custom_object_record_id, data, **kwargs):
+        "https://developer.zendesk.com/api-reference/custom-data/custom-objects/custom_object_records/#delete-custom-object-record"
+        api_path = "/api/v2/custom_objects/{custom_object_key}/records/{custom_object_record_id}"
+        api_path = api_path.format(custom_object_key=custom_object_key, custom_object_record_id=custom_object_record_id)
+        return self.call(api_path, method="DELETE", data=data, **kwargs)