Iterators: The Protocol Behind Loops
Every Python for loop uses the iterator protocol. Understanding it lets you create your own iterable objects and understand how Python works under the hood.
# The iterator protocol
# __iter__() returns an iterator object
# __next__() returns the next value, raises StopIteration when done
class CountUp:
def __init__(self, start, stop):
self.current = start
self.stop = stop
def __iter__(self):
return self # Object is its own iterator
def __next__(self):
if self.current >= self.stop:
raise StopIteration
value = self.current
self.current += 1
return value
for n in CountUp(1, 5):
print(n) # 1 2 3 4
# iter() and next() built-ins
it = iter([1, 2, 3])
next(it) # 1
next(it) # 2
next(it) # 3
# next(it) # StopIteration
Generators
Generators are functions that yield values lazily. They're the most Pythonic way to create iterators. Each yield suspends execution and resumes from that point on the next call.
# Generator function — uses yield instead of return
def count_up(start, stop):
current = start
while current < stop:
yield current # Suspends here, returns value
current += 1 # Resumes from here on next call
# Use like any iterable
for n in count_up(1, 5):
print(n) # 1 2 3 4
# Create with list()
list(count_up(1, 6)) # [1, 2, 3, 4, 5]
# Generators are lazy — they compute on demand
def fibonacci():
a, b = 0, 1
while True: # Infinite sequence!
yield a
a, b = b, a + b
fib = fibonacci()
[next(fib) for _ in range(10)]
# [0, 1, 1, 2, 3, 5, 8, 13, 21, 34]
Generator Advantages
# Memory efficiency — process 1 million lines without loading all
def read_large_file(filename):
with open(filename) as f:
for line in f:
yield line.strip()
# Process line by line — constant memory usage
for line in read_large_file("huge_log.txt"):
if "ERROR" in line:
print(line)
# Pipeline processing
def parse_numbers(lines):
for line in lines:
try:
yield float(line)
except ValueError:
pass # Skip non-numbers
def filter_positives(numbers):
for n in numbers:
if n > 0:
yield n
# Compose generators into a pipeline
lines = read_large_file("numbers.txt")
numbers = parse_numbers(lines)
positives = filter_positives(numbers)
total = sum(positives) # Processes one item at a time through the pipeline
yield from and send()
# yield from — delegate to another generator
def chain(*iterables):
for it in iterables:
yield from it # Yields all items from it
list(chain([1, 2], [3, 4], [5, 6]))
# [1, 2, 3, 4, 5, 6]
# Generator.send() — send values INTO a generator
def accumulator():
total = 0
while True:
value = yield total # Receive value, yield current total
if value is None:
break
total += value
acc = accumulator()
next(acc) # Prime the generator (advance to first yield)
acc.send(10) # 10
acc.send(20) # 30
acc.send(5) # 35
itertools: Generator Tools
# pip install — no install needed, it's in standard library!
import itertools
# Infinite iterators
itertools.count(10) # 10, 11, 12, ...
itertools.cycle("ABC") # A, B, C, A, B, C, ...
itertools.repeat(7, 3) # 7, 7, 7
# Combining
itertools.chain([1,2], [3,4]) # 1, 2, 3, 4
itertools.zip_longest("AB", "xyz", fillvalue="-") # (A,x), (B,y), (-,z)
# Filtering/grouping
itertools.takewhile(lambda x: x < 5, [1,2,3,6,1]) # 1, 2, 3
itertools.dropwhile(lambda x: x < 5, [1,2,3,6,1]) # 6, 1
itertools.islice(count(10), 5) # 10, 11, 12, 13, 14
# Combinatorics
list(itertools.permutations("ABC", 2)) # All 2-char permutations
list(itertools.combinations("ABC", 2)) # All 2-char combinations
list(itertools.product("AB", "12")) # Cartesian product
Key Takeaways
- Generators are lazy iterators: compute values on demand, memory-efficient
- yield suspends, next() resumes
- Infinite sequences: generators can yield forever — use islice to limit
- yield from: cleanly delegate to another generator
- itertools: powerful generator toolkit in the standard library
Practice Exercises
- Write a generator that yields prime numbers indefinitely.
- Write a
flatten(nested) generator that flattens arbitrarily nested lists.
- Use
itertools.groupby to group a list of sorted words by their first letter.
- Write a generator pipeline: read numbers from a file → filter out negatives → square them → yield results.