I need to read an input text file in python, by streaming line by line. That means load the text file line by line instead of all at once into memory. But my line delimiters
import re
def open_delimited(filename, delimiter, chunksize=1024, *args, **kwargs):
with open(filename, *args, **kwargs) as infile:
remainder = ''
for chunk in iter(lambda: infile.read(chunksize), ''):
pieces = re.split(delimiter, remainder+chunk)
for piece in pieces[:-1]:
yield piece
remainder = pieces[-1]
if remainder:
yield remainder
for line in open_delimited("log.txt", delimiter='/'):
print(repr(line))