Increase all read buffer sizes to 128KiB.

google · May 27, 2024 · b03ed37 · b03ed37
1 parent 7b64160
commit b03ed37
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 4 deletions.
diff --git a/internal/tailer/logstream/dgramstream.go b/internal/tailer/logstream/dgramstream.go
@@ -47,7 +47,8 @@ func (ss *dgramStream) LastReadTime() time.Time {
 	return ss.lastReadTime
 }
 
-const datagramReadBufferSize = 131071
+// The read buffer size for datagrams.
+const datagramReadBufferSize = 131072
 
 func (ss *dgramStream) stream(ctx context.Context, wg *sync.WaitGroup, waker waker.Waker) error {
 	c, err := net.ListenPacket(ss.scheme, ss.address)

diff --git a/internal/tailer/logstream/logstream.go b/internal/tailer/logstream/logstream.go
@@ -39,8 +39,10 @@ type LogStream interface {
 	IsComplete() bool        // True if the logstream has completed work and cannot recover.  The caller should clean up this logstream, creating a new logstream on a pathname if necessary.
 }
 
-// defaultReadBufferSize the size of the buffer for reading bytes into.
-const defaultReadBufferSize = 4096
+// defaultReadBufferSize the size of the buffer for reading bytes for files.
+//
+// Anecdotally the maximum file read buffer is 4GiB, but thats way too massive.
+const defaultReadBufferSize = 131072
 
 const stdinPattern = "-"
 

diff --git a/internal/tailer/logstream/pipestream.go b/internal/tailer/logstream/pipestream.go
@@ -50,7 +50,18 @@ func pipeOpen(pathname string) (*os.File, error) {
 	return os.OpenFile(pathname, os.O_RDONLY|syscall.O_NONBLOCK, 0o600) // #nosec G304 -- path already validated by caller
 }
 
-const defaultPipeReadBufferSize = 131071
+// The read buffer size for pipes.
+//
+// Before Linux 2.6.11, the capacity of a pipe was the same as the
+// system page size (e.g., 4096 bytes on i386).  Since Linux 2.6.11,
+// the pipe capacity is 16 pages (i.e., 65,536 bytes in a system
+// with a page size of 4096 bytes).  Since Linux 2.6.35, the default
+// pipe capacity is 16 pages, but the capacity can be queried and
+// set using the fcntl(2) F_GETPIPE_SZ and F_SETPIPE_SZ operations.
+// See fcntl(2) for more information.
+//
+// https://man7.org/linux/man-pages/man7/pipe.7.html
+const defaultPipeReadBufferSize = 131072
 
 func (ps *pipeStream) stream(ctx context.Context, wg *sync.WaitGroup, waker waker.Waker, _ os.FileInfo) error {
 	fd, err := pipeOpen(ps.pathname)