Learn About Delegating Generators

Introduction

Objectives:

Learn about delegating generators

Files Modified: cofollow.py, server.py

One potential issue in code that relies on generators is the problem of hiding details from the user and writing libraries. A lot of low-level mechanics are generally required to drive everything and it's often rather awkward to directly expose it to users.

Starting in Python 3.3, a new yield from statement can be used to delegate generators to another function. It is a useful way to clean-up code that relies on generators.

Skills Graph

%%%%{init: {'theme':'neutral'}}%%%% flowchart RL python(("`Python`")) -.-> python/BasicConceptsGroup(["`Basic Concepts`"]) python(("`Python`")) -.-> python/FunctionsGroup(["`Functions`"]) python(("`Python`")) -.-> python/FileHandlingGroup(["`File Handling`"]) python(("`Python`")) -.-> python/ControlFlowGroup(["`Control Flow`"]) python(("`Python`")) -.-> python/DataStructuresGroup(["`Data Structures`"]) python(("`Python`")) -.-> python/ModulesandPackagesGroup(["`Modules and Packages`"]) python(("`Python`")) -.-> python/ObjectOrientedProgrammingGroup(["`Object-Oriented Programming`"]) python(("`Python`")) -.-> python/ErrorandExceptionHandlingGroup(["`Error and Exception Handling`"]) python(("`Python`")) -.-> python/AdvancedTopicsGroup(["`Advanced Topics`"]) python(("`Python`")) -.-> python/PythonStandardLibraryGroup(["`Python Standard Library`"]) python(("`Python`")) -.-> python/NetworkingGroup(["`Networking`"]) python/BasicConceptsGroup -.-> python/comments("`Comments`") python/FunctionsGroup -.-> python/keyword_arguments("`Keyword Arguments`") python/FileHandlingGroup -.-> python/with_statement("`Using with Statement`") python/BasicConceptsGroup -.-> python/variables_data_types("`Variables and Data Types`") python/BasicConceptsGroup -.-> python/numeric_types("`Numeric Types`") python/BasicConceptsGroup -.-> python/booleans("`Booleans`") python/ControlFlowGroup -.-> python/conditional_statements("`Conditional Statements`") python/ControlFlowGroup -.-> python/for_loops("`For Loops`") python/ControlFlowGroup -.-> python/while_loops("`While Loops`") python/ControlFlowGroup -.-> python/break_continue("`Break and Continue`") python/DataStructuresGroup -.-> python/lists("`Lists`") python/DataStructuresGroup -.-> python/tuples("`Tuples`") python/FunctionsGroup -.-> python/function_definition("`Function Definition`") python/ModulesandPackagesGroup -.-> python/importing_modules("`Importing Modules`") python/ModulesandPackagesGroup -.-> python/using_packages("`Using Packages`") python/ModulesandPackagesGroup -.-> python/standard_libraries("`Common Standard Libraries`") python/ObjectOrientedProgrammingGroup -.-> python/classes_objects("`Classes and Objects`") python/ObjectOrientedProgrammingGroup -.-> python/constructor("`Constructor`") python/ObjectOrientedProgrammingGroup -.-> python/polymorphism("`Polymorphism`") python/ObjectOrientedProgrammingGroup -.-> python/encapsulation("`Encapsulation`") python/ErrorandExceptionHandlingGroup -.-> python/catching_exceptions("`Catching Exceptions`") python/ErrorandExceptionHandlingGroup -.-> python/raising_exceptions("`Raising Exceptions`") python/ErrorandExceptionHandlingGroup -.-> python/custom_exceptions("`Custom Exceptions`") python/FileHandlingGroup -.-> python/file_opening_closing("`Opening and Closing Files`") python/AdvancedTopicsGroup -.-> python/generators("`Generators`") python/PythonStandardLibraryGroup -.-> python/os_system("`Operating System and System`") python/NetworkingGroup -.-> python/socket_programming("`Socket Programming`") python/NetworkingGroup -.-> python/networking_protocols("`Networking Protocols`") python/BasicConceptsGroup -.-> python/python_shell("`Python Shell`") python/FunctionsGroup -.-> python/build_in_functions("`Build-in Functions`") subgraph Lab Skills python/comments -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/keyword_arguments -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/with_statement -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/variables_data_types -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/numeric_types -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/booleans -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/conditional_statements -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/for_loops -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/while_loops -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/break_continue -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/lists -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/tuples -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/function_definition -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/importing_modules -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/using_packages -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/standard_libraries -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/classes_objects -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/constructor -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/polymorphism -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/encapsulation -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/catching_exceptions -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/raising_exceptions -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/custom_exceptions -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/file_opening_closing -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/generators -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/os_system -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/socket_programming -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/networking_protocols -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/python_shell -.-> lab-132527{{"`Learn About Delegating Generators`"}} python/build_in_functions -.-> lab-132527{{"`Learn About Delegating Generators`"}} end

Example: Receiving messages

In Exercise 8.3, we looked at the definitions of coroutines. Coroutines were functions that you sent data to. For example:

>>> from cofollow import consumer
>>> @consumer
    def printer():
        while True:
            item = yield
            print('Got:', item)

>>> p = printer()
>>> p.send('Hello')
Got: Hello
>>> p.send('World')
Got: World
>>>

At the time, it might have been interesting to use yield to receive a value. However, if you really look at the code, it looks pretty weird--a bare yield like that? What's going on there?

In the cofollow.py file, define the following function:

def receive(expected_type):
    msg = yield
    assert isinstance(msg, expected_type), 'Expected type %s' % (expected_type)
    return msg

This function receives a message, but then verifies that it is of an expected type. Try it:

>>> from cofollow import consumer, receive
>>> @consumer
    def print_ints():
        while True:
             val = yield from receive(int)
             print('Got:', val)

>>> p = print_ints()
>>> p.send(42)
Got: 42
>>> p.send(13)
Got: 13
>>> p.send('13')
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  ...
AssertionError: Expected type <class 'int'>
>>>

From a readability point of view, the yield from receive(int) statement is a bit more descriptive--it indicates that the function will yield until it receives a message of a given type.

Now, modify all of the coroutines in coticker.py to use the new receive() function and make sure the code from Exercise 8.3 still works.

Wrapping a Socket

In the previous exercise, you wrote a simple network echo server using generators. The code for the server looked like this:

def tcp_server(address, handler):
    sock = socket(AF_INET, SOCK_STREAM)
    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
    sock.bind(address)
    sock.listen(5)
    while True:
        yield 'recv', sock
        client, addr = sock.accept()
        tasks.append(handler(client, addr))

def echo_handler(client, address):
    print('Connection from', address)
    while True:
        yield 'recv', client
        data = client.recv(1000)
        if not data:
            break
        yield 'send', client
        client.send(b'GOT:', data)
    print('Connection closed')

Create a class GenSocket that cleans up the yield statements and allows the server to be rewritten more simply as follows:

def tcp_server(address, handler):
    sock = GenSocket(socket(AF_INET, SOCK_STREAM))
    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
    sock.bind(address)
    sock.listen(5)
    while True:
        client, addr = yield from sock.accept()
        tasks.append(handler(client, addr))

def echo_handler(client, address):
    print('Connection from', address)
    while True:
        data = yield from client.recv(1000)
        if not data:
            break
        yield from client.send(b'GOT:', data)
    print('Connection closed')

Async/Await

Take the GenSocket class you just wrote and wrap all of the methods that use yield with the @coroutine decorator from the types module.

from types import coroutine
...

class GenSocket:
    def __init__(self, sock):
        self.sock = sock

    @coroutine
    def accept(self):
        yield 'recv', self.sock
        client, addr = self.sock.accept()
        return GenSocket(client), addr

    @coroutine
    def recv(self, maxsize):
        yield 'recv', self.sock
        return self.sock.recv(maxsize)

    @coroutine
    def send(self, data):
        yield 'send', self.sock
        return self.sock.send(data)

    def __getattr__(self, name):
        return getattr(self.sock, name)

Now, rewrite your server code to use async functions and await statements like this:

async def tcp_server(address, handler):
    sock = GenSocket(socket(AF_INET, SOCK_STREAM))
    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
    sock.bind(address)
    sock.listen(5)
    while True:
        client, addr = await sock.accept()
        tasks.append(handler(client, addr))

async def echo_handler(client, address):
    print('Connection from', address)
    while True:
        data = await client.recv(1000)
        if not data:
            break
        await client.send(b'GOT:', data)
    print('Connection closed')

Summary

Congratulations! You have completed the Learn About Delegating Generators lab. You can practice more labs in LabEx to improve your skills.