How to paginate Organizations:ListAccountsForParent?

Moto3 doesn’t yet paginate the response from Organizations list_accounts_for_parent.

I need to paginate this API in my application and I’d like to test it with moto.

I tried to implement it myself by adapting the changes for the list_accounts API made recently in #4951.

My implmenetation is failing to iterate beyond the first page.

Below I’ve included a Pytest suite with my implementation and some tests.

I expect all the tests to pass.

Instead test_with_pagination_list_paginator_all_pages_have_1_account fails with the following error:

botocore.exceptions.ClientError: An error occurred (InvalidToken) when calling the ListAccountsForParent operation: Invalid Token: Input inconsistent with page token: {'parameterChecksum': -953616664862312029, 'uniqueAttributes': '1650119718.751258'}

What’s missing to make this work?

Pytest suite:

from boto3 import Session
import json
from typing import Iterator
from pytest import fixture, mark
from pytest_mock import MockerFixture
from moto.utilities.paginator import paginate
from moto import mock_organizations


def _create_org(session: Session) -> None:
    org = session.client("organizations")
    org.create_organization(FeatureSet="ALL")


def _create_member(session: Session) -> None:
    org = session.client("organizations")
    org.create_account(Email="[e-mail]", AccountName="Example")


@fixture()
def session() -> Iterator[Session]:
    with mock_organizations():
        yield Session()


@fixture()
def paginated_moto(mocker: MockerFixture) -> Iterator[None]:

    model = {
        "list_accounts_for_parent": {
            "input_token": "next_token",
            "limit_key": "max_results",
            "limit_default": 1,
            "result_key": "Accounts",
            "unique_attribute": "JoinedTimestamp",
        }
    }

    @paginate(pagination_model=model)
    def list_accounts_for_parent(self, **kwargs):
        parent_id = self.validate_parent_id(kwargs["ParentId"])
        return [
            account.describe()
            for account in self.accounts
            if account.parent_id == parent_id
        ]
    
    def response_list_accounts_for_parent(self):
        max_results = self._get_int_param("MaxResults")
        next_token = self._get_param("NextToken")
        accounts, next_token = self.organizations_backend.list_accounts_for_parent(
            max_results=max_results, next_token=next_token, **self.request_params
        )
        response = {"Accounts": accounts}
        if next_token:
            response["NextToken"] = next_token
        return json.dumps(response)

    mocker.patch("moto.organizations.utils.PAGINATION_MODEL", model)

    # The mock name must match the key in the pagination model.
    mocker.patch(
        "moto.organizations.models.OrganizationsBackend.list_accounts_for_parent",
        list_accounts_for_parent
    )
    mocker.patch(
        "moto.organizations.responses.OrganizationsResponse.list_accounts_for_parent",
        response_list_accounts_for_parent
    )


@fixture(autouse=True)
def org(session: Session):
    _create_org(session)
    _create_member(session)


def test_without_pagination_list_returns_2_accounts(session: Session):
    org = session.client("organizations")
    root_id = org.list_roots()["Roots"][0]["Id"]
    accounts = org.list_accounts_for_parent(ParentId=root_id)["Accounts"]
    assert len(accounts) == 2


@mark.usefixtures("paginated_moto")
def test_with_pagination_list_returns_1_account(session: Session):
    org = session.client("organizations")
    root_id = org.list_roots()["Roots"][0]["Id"]
    accounts = org.list_accounts_for_parent(ParentId=root_id)["Accounts"]
    assert len(accounts) == 1


@mark.usefixtures("paginated_moto")
def test_with_pagination_list_paginator_first_page_has_1_account(session: Session):
    org = session.client("organizations")
    root_id = org.list_roots()["Roots"][0]["Id"]

    paginator = org.get_paginator("list_accounts_for_parent")
    accounts = next(iter(paginator.paginate(ParentId=root_id)))["Accounts"]
    assert len(accounts) == 1


@mark.usefixtures("paginated_moto")
def test_with_pagination_list_paginator_all_pages_have_1_account(session: Session):
    org = session.client("organizations")
    root_id = org.list_roots()["Roots"][0]["Id"]

    paginator = org.get_paginator("list_accounts_for_parent")
    for page in paginator.paginate(ParentId=root_id):
        accounts = page["Accounts"]
        assert len(accounts) == 1

Complete pytest output:

$ poetry run pytest test_moto_pagination.py
========================================== test session starts ==========================================
platform linux -- Python 3.8.10, pytest-7.1.1, pluggy-1.0.0
rootdir: /home/isme/Repos/Personal/github/aws-org-tree
plugins: mock-3.7.0
collected 4 items                                                                                       

test_moto_pagination.py ...F                                                                      [100%]

=============================================== FAILURES ================================================
_____________________ test_with_pagination_list_paginator_all_pages_have_1_account ______________________

session = Session(region_name=None)

    @mark.usefixtures("paginated_moto")
    def test_with_pagination_list_paginator_all_pages_have_1_account(session: Session):
        org = session.client("organizations")
        root_id = org.list_roots()["Roots"][0]["Id"]
    
        paginator = org.get_paginator("list_accounts_for_parent")
>       for page in paginator.paginate(ParentId=root_id):

test_moto_pagination.py:109: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/home/isme/.cache/pypoetry/virtualenvs/aws-org-tree-bM0Zwf-S-py3.8/lib/python3.8/site-packages/botocore/paginate.py:255: in __iter__
    response = self._make_request(current_kwargs)
/home/isme/.cache/pypoetry/virtualenvs/aws-org-tree-bM0Zwf-S-py3.8/lib/python3.8/site-packages/botocore/paginate.py:334: in _make_request
    return self._method(**current_kwargs)
/home/isme/.cache/pypoetry/virtualenvs/aws-org-tree-bM0Zwf-S-py3.8/lib/python3.8/site-packages/botocore/client.py:388: in _api_call
    return self._make_api_call(operation_name, kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <botocore.client.Organizations object at 0x7f71217a23d0>, operation_name = 'ListAccountsForParent'
api_params = {'NextToken': 'eyJwYXJhbWV0ZXJDaGVja3N1bSI6IC05NTM2MTY2NjQ4NjIzMTIwMjksICJ1bmlxdWVBdHRyaWJ1dGVzIjogIjE2NTAxMTk3MTguNzUxMjU4In0=', 'ParentId': 'r-hiyc'}

    def _make_api_call(self, operation_name, api_params):
        operation_model = self._service_model.operation_model(operation_name)
        service_name = self._service_model.service_name
        history_recorder.record('API_CALL', {
            'service': service_name,
            'operation': operation_name,
            'params': api_params,
        })
        if operation_model.deprecated:
            logger.debug('Warning: %s.%s() is deprecated',
                         service_name, operation_name)
        request_context = {
            'client_region': self.meta.region_name,
            'client_config': self.meta.config,
            'has_streaming_input': operation_model.has_streaming_input,
            'auth_type': operation_model.auth_type,
        }
        request_dict = self._convert_to_request_dict(
            api_params, operation_model, context=request_context)
    
        service_id = self._service_model.service_id.hyphenize()
        handler, event_response = self.meta.events.emit_until_response(
            'before-call.{service_id}.{operation_name}'.format(
                service_id=service_id,
                operation_name=operation_name),
            model=operation_model, params=request_dict,
            request_signer=self._request_signer, context=request_context)
    
        if event_response is not None:
            http, parsed_response = event_response
        else:
            http, parsed_response = self._make_request(
                operation_model, request_dict, request_context)
    
        self.meta.events.emit(
            'after-call.{service_id}.{operation_name}'.format(
                service_id=service_id,
                operation_name=operation_name),
            http_response=http, parsed=parsed_response,
            model=operation_model, context=request_context
        )
    
        if http.status_code >= 300:
            error_code = parsed_response.get("Error", {}).get("Code")
            error_class = self.exceptions.from_code(error_code)
>           raise error_class(parsed_response, operation_name)
E           botocore.exceptions.ClientError: An error occurred (InvalidToken) when calling the ListAccountsForParent operation: Invalid Token: Input inconsistent with page token: {'parameterChecksum': -953616664862312029, 'uniqueAttributes': '1650119718.751258'}

/home/isme/.cache/pypoetry/virtualenvs/aws-org-tree-bM0Zwf-S-py3.8/lib/python3.8/site-packages/botocore/client.py:708: ClientError
======================================== short test summary info ========================================
FAILED test_moto_pagination.py::test_with_pagination_list_paginator_all_pages_have_1_account - botocor...
====================================== 1 failed, 3 passed in 0.57s ======================================

Moto version: 3.1.4 Boto3 version: 1.18.58

[bpandola]

Apologies, @iainelder. I was doing pagination for another Organizations action and thought I’d do this one too while I was at it. Didn’t mean to step on any toes!

[iainelder]

@bpandola , no problem, thanks for keeping the momentum going! I had let this one slip because I was exploring how to add type annotatations as @bblommers mentioned.

@bblommers, thanks for the reminder about that. I’ve been away from the keyboard for a while, but now that I’m back I’ll try to get a draft PR together for the type annotations. There are some tricky problems to solve there 😃