Optimizes iter_lines handling long lines

botocore/response.py

@@ -54,6 +54,7 @@ def __init__(self, raw_stream, content_length):
         self._raw_stream = raw_stream
         self._content_length = content_length
         self._amount_read = 0
+        self._pending_line_data = []
 
     def __del__(self):
         # Extending destructor in order to preserve the underlying raw_stream.
@@ -138,14 +139,28 @@ def iter_lines(self, chunk_size=_DEFAULT_CHUNK_SIZE, keepends=False):
         This is achieved by reading chunk of bytes (of size chunk_size) at a
         time from the raw stream, and then yielding lines from there.
         """
-        pending = b''
+
+        def _maybe_trim(line):
+            if keepends or not line.endswith(b'\n'):
+                return line
+            return line[:-2] if line.endswith(b'\r\n') else line[:-1]
+
+        def _flush(final_segment):
+            line = b''.join(self._pending_line_data + [final_segment])
+            self._pending_line_data = []
+            return _maybe_trim(line)
+
         for chunk in self.iter_chunks(chunk_size):
-            lines = (pending + chunk).splitlines(True)
-            for line in lines[:-1]:
-                yield line.splitlines(keepends)[0]
-            pending = lines[-1]
-        if pending:
-            yield pending.splitlines(keepends)[0]
+            lines = chunk.splitlines(True)
+            if len(lines) > 1:
+                for segment in lines[0:-1]:
+                    yield _flush(segment)
+            if lines[-1].endswith(b'\n'):
+                yield _flush(lines[-1])
+            else:
+                self._pending_line_data += lines[-1:]
+        if last := _flush(b''):
+            yield last
 
     def iter_chunks(self, chunk_size=_DEFAULT_CHUNK_SIZE):
         """Return an iterator to yield chunks of chunk_size bytes from the raw