diff --git a/.gitignore b/.gitignore index af4f2e6..229e96f 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,6 @@ target/ .idea/ .DS_Store + +# test config +tests/zdtestcfg.py \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e137fad --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dde6ee..17ed6db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 2.8.6 +- Add support for cursor based pagination and make it default +- Allow disabling cursor based pagination, because certain endpoints don't support it yet. +- Add support for incremental api cursor and time pagination. + ## 2.8.0 - Regenerate API from updated mirror. see [full commit](https://github.com/fprimex/zdesk/commit/4982b3dad9581fbb49d71307abc229dc4169ab74). diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 978ae56..8507d73 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -9,6 +9,7 @@ * Dominik Miedziński (Booksy International Sp. z o. o.) * Sarfaraz Soomro (Incremental ticket pagination) * Craig Davis (Major `api_gen` updates for zdesk 2.6.0) +* Otto Tamas (Pagination improvements - zdesk 2.8.6) ## zendesk and zdesk 1.x diff --git a/README.md b/README.md index 6e42e62..13de0ad 100644 --- a/README.md +++ b/README.md @@ -154,19 +154,24 @@ equal to (the integer) `201`. ## Getting all pages -There is a common pattern where a request will return one page of data along -with a `next_page` location. In order to retrieve all results, it is necessary -to continue retrieving every `next_page` location. The results then all need to -be processed together. A loop to get all pages ends up stamped throughout -Zendesk code, since many API methods return paged lists of objects. - -As a convenience, passing `get_all_pages` to any API method will do this for -you, and will also merge all responses. The result is a single, large object -that appears to be the result of one single call. The logic for this -combination and reduction is well documented in the -[source](https://github.com/fprimex/zdesk/blob/master/zdesk/zdesk.py#L534) -(look for the line reading `Now we need to try to combine or reduce the -results`, if the line number has shifted since this writing). +Some endpoints support pagination in order to efficiently serve a larger +amount of data. As a convenience, passing `get_all_pages` to endpoints +that support pagination will follow links to progressively fetch all +data. The result is a single, large object that appears to be the result +of one single call. + +- by default, cursor based pagination is used as recommended by the +zdesk API documentation to favor efficiency. +- you can optionally opt for falling back to offset based pagination by +passing `cursor_pagination=False` flag. +- a limited number of endpoints don't support cursor based pagination yet. +If you get errors saying that `page[size]` parameter is not supported, then +try setting `cursor_pagination=False`. Note that offset based pagination now +has a hard limit (usually 10k) on the returned number of items, so I discourage +using it for endpoints that support cursor based pagination. +- the incremental API has different mechanics for pagination, which are also +supported by this lib. `cursor_pagination=False` might be required for most +incremental API endpoints. ## MIME types for data diff --git a/setup.py b/setup.py index 0e75d38..b1329ad 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ # Basic package information. name = 'zdesk', author = 'Brent Woodruff', - version = '2.8.0', + version = '2.8.8', author_email = 'brent@fprimex.com', packages = ['zdesk'], include_package_data = True, diff --git a/tests/paginate/test_cursor_pagination.py b/tests/paginate/test_cursor_pagination.py new file mode 100644 index 0000000..c92d3be --- /dev/null +++ b/tests/paginate/test_cursor_pagination.py @@ -0,0 +1,27 @@ +import pytest + +from zdesk import Zendesk + +@pytest.fixture(scope="module", autouse=True) +def expected_ticket_count(zd: Zendesk): + response = zd.tickets_count_list() + count = response["count"]["value"] + assert count >= 100, "Please create at least 100 tickets for this test to run" + return count + + +def test_ticket_cursor_pagination(zd: Zendesk, expected_ticket_count: int): + response = zd.tickets_list(get_all_pages=True) + tickets = response["tickets"] + assert len(tickets) == expected_ticket_count + +def test_ticket_cursor_pagination_custom_page_size(zd: Zendesk, expected_ticket_count: int): + response = zd.tickets_list(page_size=50, get_all_pages=True) + tickets = response["tickets"] + assert len(tickets) == expected_ticket_count + + +def test_ticket_offset_pagination(zd: Zendesk, expected_ticket_count: int): + response = zd.tickets_list(per_page=100, get_all_pages=True, cursor_pagination=False) + tickets = response["tickets"] + assert len(tickets) == expected_ticket_count diff --git a/tests/paginate/test_incremental_api_pagination.py b/tests/paginate/test_incremental_api_pagination.py new file mode 100644 index 0000000..c971cda --- /dev/null +++ b/tests/paginate/test_incremental_api_pagination.py @@ -0,0 +1,23 @@ +import datetime + +from zdesk import Zendesk + +days_ago = 60 +start_time = (datetime.datetime.now() - datetime.timedelta(days=days_ago)).timestamp() +incremental_api_page_size = 1000 + +def test_incremental_ticket_pagination(zd: Zendesk): + response = zd.incremental_tickets_list(get_all_pages=True, cursor_pagination=False, start_time=start_time) + time_pagination_tickets = response["tickets"] + assert response["end_time"] > start_time + assert response["end_of_stream"] is True + assert len(time_pagination_tickets) > incremental_api_page_size + + response = zd.incremental_tickets_cursor_list(get_all_pages=True, cursor_pagination=False, start_time=start_time) + cursor_pagination_tickets = response["tickets"] + assert response["end_of_stream"] is True + + # time pagination provides some duplicates + time_pagination_ticket_ids = {ticket["id"] for ticket in time_pagination_tickets} + cursor_pagination_tickets_ids = {ticket["id"] for ticket in cursor_pagination_tickets} + assert time_pagination_ticket_ids == cursor_pagination_tickets_ids diff --git a/tests/paginate/test_offset_pagination.py b/tests/paginate/test_offset_pagination.py new file mode 100644 index 0000000..d2f406c --- /dev/null +++ b/tests/paginate/test_offset_pagination.py @@ -0,0 +1,29 @@ +import pytest + +from zdesk import Zendesk, ZendeskError + +@pytest.fixture(scope="module", autouse=True) +def expected_ticket_count(zd: Zendesk): + response = zd.search_count(query="*") + count = response["count"] + assert count >= 100, "Please create at least 100 tickets for this test to run" + return count + + +def test_search_offset_pagination(zd: Zendesk, expected_ticket_count: int): + # Try to implement a way to return partial results. + # This includes refactoring, because results are merged together at the end + # of pagination. + try: + response = zd.search(query="*", get_all_pages=True, cursor_pagination=False) + except ZendeskError as e: + assert len(e.partial_results["results"]) == 1000 + assert e.partial_results["count"] == expected_ticket_count + else: + assert len(response["results"]) == expected_ticket_count + +def test_search_with_cursor_pagination_is_unsupported(zd: Zendesk): + with pytest.raises(ZendeskError) as e: + zd.search(query="*", get_all_pages=True, cursor_pagination=True) + + assert e.value.partial_results is None \ No newline at end of file diff --git a/zdesk/zdesk.py b/zdesk/zdesk.py index 9f41cbc..9e9974b 100644 --- a/zdesk/zdesk.py +++ b/zdesk/zdesk.py @@ -72,10 +72,11 @@ def get_id_from_url(url): class ZendeskError(Exception): - def __init__(self, msg, code, response): + def __init__(self, msg, code, response, partial_results = None): self.msg = msg self.error_code = code self.response = response + self.partial_results = partial_results def __str__(self): return repr('%s: %s %s' % (self.error_code, self.msg, self.response)) @@ -309,7 +310,7 @@ def max_retries(self): def call(self, path, query=None, method='GET', data=None, files=None, get_all_pages=False, complete_response=False, retry_on=None, max_retries=0, raw_query=None, retval=None, - **kwargs): + cursor_pagination=True, **kwargs): """Make a REST call to the Zendesk web service. Parameters: @@ -333,6 +334,10 @@ def call(self, path, query=None, method='GET', data=None, appended to the URL path and will completely override / discard any other query parameters. Enables use cases where query parameters need to be repeated in the query string. + cursor_pagination - Whether to use cursor-based pagination or not. + Defaults to True. Some endpoints don't support cursor-based + pagination yet, so you can set this to False to revert to + offset/page based pagination. retval - Request a specific part of the returned response. Valid values are 'content', 'code', 'location', and 'headers'. JSON content is still automatically deserialized if possible. @@ -373,6 +378,9 @@ def call(self, path, query=None, method='GET', data=None, else: kwargs = query + if get_all_pages and cursor_pagination: + kwargs['page[size]'] = kwargs.pop('per_page', 100) + if raw_query: path = path + raw_query kwargs = None @@ -434,47 +442,26 @@ def call(self, path, query=None, method='GET', data=None, code = response.status_code try: - if not 200 <= code < 300 and code != 422: + if not 200 <= code < 300: + partial_results = self._combine_results(results) if results else None if code == 401: raise AuthenticationError( - response.content, code, response) + response.content, code, response, partial_results) elif code == 429: raise RateLimitError( - response.content, code, response) + response.content, code, response, partial_results) else: raise ZendeskError( - response.content, code, response) + response.content, code, response, partial_results) except ZendeskError: - if request_count <= self.max_retries: + if request_count <= self.max_retries and code != 422: self._handle_retry(response) continue else: raise - # Deserialize json content if content exists. - # In some cases Zendesk returns ' ' strings. - # Also return false non strings (0, [], (), {}) - if response.content.strip() and 'json' in response.headers['content-type']: - content = response.json() - - # set url to the next page if that was returned in the response - url = content.get('next_page', None) - # url we get above already has the start_time appended to it, - # specific to incremental exports - kwargs = {} - elif response.content.strip() and 'text' in response.headers['content-type']: - try: - content = response.json() - # set url to the next page if that was returned in the response - url = content.get('next_page', None) - # url we get above already has the start_time appended to it, - # specific to incremental exports - kwargs = {} - except ValueError: - content = response.content - else: - content = response.content - url = None + + content, url, kwargs = self._parse_response(response, kwargs) if complete_response: results.append({ @@ -519,6 +506,7 @@ def call(self, path, query=None, method='GET', data=None, # also note that incremental/ticket_metric_events end-point has a 10,000 items per page limit url = None if (url is not None and 'incremental' in url and + 'count' in content and content.get('count') < 1000) else url all_requests_complete = not (get_all_pages and url) request_count = 0 @@ -553,7 +541,47 @@ def call(self, path, query=None, method='GET', data=None, # we have a list of simple objects like strings, but they are not # all the same so send them all back. return results + + return self._combine_results(results) + + def _parse_response(self, response, kwargs): + # Deserialize json content if content exists. + # In some cases Zendesk returns ' ' strings. + # Also return false non strings (0, [], (), {}) + if not response.content.strip(): + return response.content, None, kwargs + + if 'json' in response.headers['content-type']: + json = response.json() + elif 'text' in response.headers['content-type']: + try: + json = response.json() + except ValueError: + json = None + + if json: + url = None + + # set url to the next page if that was returned in the response + if {'meta', 'links'} <= json.keys(): + # cursor based pagination + if json.get('meta', {}).get('has_more'): + url = json.get('links', {}).get('next') + else: + # offset based pagination + if not json.get('end_of_stream', False): + url = json.get('next_page', None) + + # incremental api cursor pagination uses after_url instead + url = url if url else json.get('after_url', None) + + # url we get above already has kwargs appended, + return json, url, {} + + return response.content, None, kwargs + + def _combine_results(self, results): # may have a sequence of response contents # (dicts, possibly lists in the future as that is valid json also) combined_dict_results = {} diff --git a/zdesk/zdesk_api.py b/zdesk/zdesk_api.py index aaa5ead..aa7c26a 100644 --- a/zdesk/zdesk_api.py +++ b/zdesk/zdesk_api.py @@ -2539,6 +2539,11 @@ def help_center_sections_list(self, locale=None, **kwargs): api_path = api_opt_path.format(locale=locale) return self.call(api_path, **kwargs) + def help_center_integration_keys(self, **kwargs): + "https://developer.zendesk.com/api-reference/help_center/help-center-api/help_center_jwts/#list-public-keys" + api_path = "/api/v2/help_center/integration/keys" + return self.call(api_path, **kwargs) + def help_center_translation_delete(self, translation_id, **kwargs): "https://developer.zendesk.com/api-reference/help_center/help-center-api/translations#delete-translation" api_path = "/api/v2/help_center/translations/{translation_id}" @@ -4586,6 +4591,19 @@ def tickets_update_many(self, data, **kwargs): api_path = "/api/v2/tickets/update_many" return self.call(api_path, method="PUT", data=data, **kwargs) + def tickets_update_many_by_fields_dict(self, data, ids=None, **kwargs): + "https://developer.zendesk.com/rest_api/docs/core/tickets#update-many-tickets" + api_path = "/api/v2/tickets/update_many.json" + api_query = {} + if "query" in kwargs.keys(): + api_query.update(kwargs["query"]) + del kwargs["query"] + if ids: + api_query.update({ + "ids": ids, + }) + return self.call(api_path, query=api_query, method="PUT", data=data, **kwargs) + def trigger_categories_job_create(self, data, **kwargs): "https://developer.zendesk.com/api-reference/ticketing/trigger_categories#create-batch-job-for-trigger-categories" api_path = "/api/v2/trigger_categories/jobs" @@ -5149,10 +5167,22 @@ def users_request_create(self, data, **kwargs): api_path = "/api/v2/users/request_create" return self.call(api_path, method="POST", data=data, **kwargs) - def users_search(self, **kwargs): - "https://developer.zendesk.com/api-reference/ticketing/users#search-users" - api_path = "/api/v2/users/search" - return self.call(api_path, **kwargs) + def users_search(self, external_id=None, query=None, **kwargs): + "https://developer.zendesk.com/rest_api/docs/core/users#search-users" + api_path = "/api/v2/users/search.json" + api_query = {} + if "query" in kwargs.keys(): + api_query.update(kwargs["query"]) + del kwargs["query"] + if external_id: + api_query.update({ + "external_id": external_id, + }) + if query: + api_query.update({ + "query": query, + }) + return self.call(api_path, query=api_query, **kwargs) def users_show_many(self, **kwargs): "https://developer.zendesk.com/api-reference/ticketing/users#show-many-users" @@ -5373,4 +5403,21 @@ def workspaces_reorder(self, data, **kwargs): api_path = "/api/v2/workspaces/reorder" return self.call(api_path, method="PUT", data=data, **kwargs) + def custom_object_record_create(self, custom_object_key, data, **kwargs): + "https://developer.zendesk.com/api-reference/custom-data/custom-objects/custom_object_records/#create-custom-object-record" + api_path = "/api/v2/custom_objects/{custom_object_key}/records" + api_path = api_path.format(custom_object_key=custom_object_key) + return self.call(api_path, method="POST", data=data, **kwargs) + + def custom_object_record_show(self, custom_object_key, custom_object_record_id, data, **kwargs): + "https://developer.zendesk.com/api-reference/custom-data/custom-objects/custom_object_records/#show-custom-object-record" + api_path = "/api/v2/custom_objects/{custom_object_key}/records/{custom_object_record_id}" + api_path = api_path.format(custom_object_key=custom_object_key, custom_object_record_id=custom_object_record_id) + return self.call(api_path, method="GET", data=data, **kwargs) + + def custom_object_record_delete(self, custom_object_key, custom_object_record_id, data, **kwargs): + "https://developer.zendesk.com/api-reference/custom-data/custom-objects/custom_object_records/#delete-custom-object-record" + api_path = "/api/v2/custom_objects/{custom_object_key}/records/{custom_object_record_id}" + api_path = api_path.format(custom_object_key=custom_object_key, custom_object_record_id=custom_object_record_id) + return self.call(api_path, method="DELETE", data=data, **kwargs)