[go: nahoru, domu]
Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enable use of page_size for clients #408

Merged
merged 2 commits into from
Dec 10, 2014
Merged

Enable use of page_size for clients #408

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
86ce5fd
Enable use of page_size for clients
kyleknap Dec 9, 2014
b876203
Add a todo commnet for the test
kyleknap Dec 10, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 16 additions & 11 deletions botocore/paginate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,16 @@

class PageIterator(object):
def __init__(self, method, input_token, output_token, more_results,
result_keys, non_aggregate_keys, max_items, starting_token,
page_size, op_kwargs):
result_keys, non_aggregate_keys, limit_key, max_items,
starting_token, page_size, op_kwargs):
self._method = method
self._op_kwargs = op_kwargs
self._input_token = input_token
self._output_token = output_token
self._more_results = more_results
self._result_keys = result_keys
self._max_items = max_items
self._limit_key = limit_key
self._starting_token = starting_token
self._page_size = page_size
self._op_kwargs = op_kwargs
Expand Down Expand Up @@ -135,11 +136,9 @@ def _inject_starting_params(self, op_kwargs):
next_token = self._parse_starting_token()[0]
self._inject_token_into_kwargs(op_kwargs, next_token)
if self._page_size is not None:
# Determine the proper parameter name for limiting
# page size.
page_size_name = self._operation.pagination['limit_key']
# Pass the page size as the determined parameter name.
op_kwargs[page_size_name] = self._page_size
# Pass the page size as the parameter name for limiting
# page size, also known as the limit_key.
op_kwargs[self._limit_key] = self._page_size

def _inject_token_into_kwargs(self, op_kwargs, next_token):
for name, token in zip(self._input_token, next_token):
Expand Down Expand Up @@ -268,6 +267,7 @@ def __init__(self, method, pagination_config):
self._non_aggregate_keys = self._get_non_aggregate_keys(
self._pagination_cfg)
self._result_keys = self._get_result_keys(self._pagination_cfg)
self._limit_key = self._get_limit_key(self._pagination_cfg)

@property
def result_keys(self):
Expand Down Expand Up @@ -307,6 +307,9 @@ def _get_result_keys(self, config):
result_key = [jmespath.compile(rk) for rk in result_key]
return result_key

def _get_limit_key(self, config):
return config.get('limit_key')

def paginate(self, **kwargs):
"""Create paginator object for an operation.

Expand All @@ -320,6 +323,7 @@ def paginate(self, **kwargs):
self._method, self._input_token,
self._output_token, self._more_results,
self._result_keys, self._non_aggregate_keys,
self._limit_key,
page_params['max_items'],
page_params['starting_token'],
page_params['page_size'],
Expand Down Expand Up @@ -373,12 +377,12 @@ def __iter__(self):
class DeprecatedPageIterator(PageIterator):
def __init__(self, operation, endpoint, input_token,
output_token, more_results,
result_keys, non_aggregate_keys, max_items, starting_token,
page_size, op_kwargs):
result_keys, non_aggregate_keys, limit_key, max_items,
starting_token, page_size, op_kwargs):
super(DeprecatedPageIterator, self).__init__(
None, input_token, output_token, more_results, result_keys,
non_aggregate_keys, max_items, starting_token, page_size,
op_kwargs)
non_aggregate_keys, limit_key, max_items,
starting_token, page_size, op_kwargs)
self._operation = operation
self._endpoint = endpoint

Expand Down Expand Up @@ -410,6 +414,7 @@ def paginate(self, endpoint, **kwargs):
self._operation, endpoint, self._input_token,
self._output_token, self._more_results,
self._result_keys, self._non_aggregate_keys,
self._limit_key,
page_params['max_items'],
page_params['starting_token'],
page_params['page_size'],
Expand Down
20 changes: 19 additions & 1 deletion tests/integration/test_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ class BaseS3Test(unittest.TestCase):
def setUp(self):
self.session = botocore.session.get_session()
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
self.region = 'us-east-1'
self.endpoint = self.service.get_endpoint(self.region)
self.keys = []

def create_object(self, key_name, body='foo'):
Expand Down Expand Up @@ -204,6 +205,23 @@ def test_can_paginate_with_page_size(self):
for el in data]
self.assertEqual(key_names, ['key0', 'key1', 'key2', 'key3', 'key4'])

def test_client_can_paginate_with_page_size(self):
for i in range(5):
key_name = 'key%s' % i
self.create_object(key_name)
# Eventual consistency.
time.sleep(3)
client = self.session.create_client('s3', region_name=self.region)
paginator = client.get_paginator('list_objects')
generator = paginator.paginate(page_size=1,
Bucket=self.bucket_name)
responses = list(generator)
self.assertEqual(len(responses), 5, responses)
data = [r for r in responses]
key_names = [el['Contents'][0]['Key']
for el in data]
self.assertEqual(key_names, ['key0', 'key1', 'key2', 'key3', 'key4'])

def test_result_key_iters(self):
for i in range(5):
key_name = 'key/%s/%s' % (i, i)
Expand Down
35 changes: 35 additions & 0 deletions tests/unit/test_paginate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,17 @@
# language governing permissions and limitations under the License.

from tests import unittest
from botocore.paginate import Paginator as FuturePaginator
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since the Paginator name is clobbered by the renaming of DeprecatedPaginator, I had to rename the paginator that we will eventually be using. I considered switching everything over to the paginator will be using in the future. However the CLI relies on the deprecated paginators, so I was hesitant to make the switch.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be nice to add this as a comment in the test so we don't forget why this was done.

from botocore.paginate import DeprecatedPaginator as Paginator
from botocore.exceptions import PaginationError
from botocore.operation import Operation

import mock

# TODO: FuturePaginator tests should be merged into tests that used the renamed
# Deprecated paginators when we completely remove the Deprecated
# paginator class and make all of the tests use the actual Paginator class


class TestPagination(unittest.TestCase):
def setUp(self):
Expand Down Expand Up @@ -149,6 +154,36 @@ def test_more_tokens_is_path_expression(self):
mock.call(None, NextToken='token1'),])


class TestFuturePaginator(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users",
"limit_key": "MaxKeys",
}

self.paginator = FuturePaginator(self.method, self.paginate_config)

def test_with_page_size(self):
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate(page_size=1):
users += page['Users']
self.assertEqual(
self.method.call_args_list,
[mock.call(MaxKeys=1),
mock.call(Marker='m1', MaxKeys=1),
mock.call(Marker='m2', MaxKeys=1)]
)


class TestPaginatorObjectConstruction(unittest.TestCase):
def test_pagination_delegates_to_paginator(self):
paginator_cls = mock.Mock()
Expand Down