diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 7d453d7c003b..d12445e3907a 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -213,31 +213,60 @@ ext4_extending_io(struct inode *inode, loff_t offset, size_t len) return false; } -/* Is IO overwriting allocated or initialized blocks? */ -static bool ext4_overwrite_io(struct inode *inode, - loff_t pos, loff_t len, bool *unwritten) +/* + * Does an unaligned DIO write require partial block zeroing? + * + * Partial block zeroing is performed only for the head and tail blocks + * when they are partially covered by the write and the underlying extent + * is a hole or unwritten. Middle blocks (fully covered by the write) + * are written as whole blocks without zeroing. + * + * When zeroing is required, two concurrent unaligned DIO writes to the + * same partial block can race and corrupt each other's data, so the + * caller must take the exclusive i_rwsem and drain in-flight DIO. When + * zeroing is not required, shared lock is safe -- block allocation and + * unwritten conversion for middle blocks are protected by i_data_sem + * and inode_dio_begin(). + */ +static bool ext4_dio_needs_zeroing(struct inode *inode, loff_t pos, loff_t len) { struct ext4_map_blocks map; unsigned int blkbits = inode->i_blkbits; - int err, blklen; + unsigned long blockmask = inode->i_sb->s_blocksize - 1; + bool head_partial, tail_partial; + ext4_lblk_t head_lblk, tail_lblk; + int err; if (pos + len > i_size_read(inode)) - return false; + return true; - map.m_lblk = pos >> blkbits; - map.m_len = EXT4_MAX_BLOCKS(len, pos, blkbits); - blklen = map.m_len; + head_partial = (pos & blockmask) != 0; + tail_partial = ((pos + len) & blockmask) != 0; + head_lblk = pos >> blkbits; + tail_lblk = (pos + len - 1) >> blkbits; - err = ext4_map_blocks(NULL, inode, &map, 0); - if (err != blklen) - return false; - /* - * 'err==len' means that all of the blocks have been preallocated, - * regardless of whether they have been initialized or not. We need to - * check m_flags to distinguish the unwritten extents. - */ - *unwritten = !(map.m_flags & EXT4_MAP_MAPPED); - return true; + /* Check the head partial block. */ + if (head_partial) { + map.m_lblk = head_lblk; + map.m_len = tail_lblk - head_lblk + 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + /* If this mapping already covers the tail block, we're done. */ + if (!tail_partial || map.m_lblk + err > tail_lblk) + return false; + } + + /* Check the tail partial block. */ + if (tail_partial) { + map.m_lblk = tail_lblk; + map.m_len = 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + } + + return false; } static ssize_t ext4_generic_write_checks(struct kiocb *iocb, @@ -453,9 +482,10 @@ static const struct iomap_dio_ops ext4_dio_write_ops = { * i_data_sem serializes concurrent extent tree modifications. * * 4. Otherwise, the write is unaligned and non-extending. Shared lock is - * only safe for pure written-extent overwrites. Unwritten extents or - * holes require exclusive lock because concurrent partial block zeroing - * in the DIO layer could corrupt data. + * safe unless the DIO layer needs to perform partial block zeroing -- + * i.e. the head or tail partial block sits on a hole or unwritten + * extent. In that case upgrade to the exclusive lock and drain + * in-flight DIO to avoid races with concurrent partial block zeroing. */ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, bool *ilock_shared, bool *extend, @@ -466,7 +496,7 @@ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, loff_t offset; size_t count; ssize_t ret; - bool overwrite = true, unaligned_io, unwritten = false; + bool needs_zeroing = false; restart: ret = ext4_generic_write_checks(iocb, from); @@ -476,21 +506,22 @@ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, offset = iocb->ki_pos; count = ret; - unaligned_io = ext4_unaligned_io(inode, from, offset); *extend = ext4_extending_io(inode, offset, count); /* - * For unaligned writes we need to know the extent state to determine - * whether shared lock is safe. For aligned writes we skip this check - * entirely since allocation under shared lock is safe. + * For unaligned writes, check whether partial block zeroing will be + * needed. If so, exclusive lock is required to serialize against + * concurrent DIO that could race with the zeroing. + * + * For aligned writes we skip this check entirely since allocation + * under shared lock is safe. */ - if (unaligned_io) - overwrite = ext4_overwrite_io(inode, offset, count, &unwritten); + if (ext4_unaligned_io(inode, from, offset)) + needs_zeroing = ext4_dio_needs_zeroing(inode, offset, count); /* Determine whether we need to upgrade to an exclusive lock. */ if (*ilock_shared && - ((!IS_NOSEC(inode) || *extend || - (unaligned_io && (!overwrite || unwritten))))) { + (!IS_NOSEC(inode) || *extend || needs_zeroing)) { if (iocb->ki_flags & IOCB_NOWAIT) { ret = -EAGAIN; goto out; @@ -504,16 +535,23 @@ static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, /* * Now that locking is settled, determine dio flags and exclusivity * requirements. We don't use DIO_OVERWRITE_ONLY because we enforce - * behavior already. The inode lock is already held exclusive if the - * write is unaligned non-overwrite or extending, so drain all - * outstanding dio and set the force wait dio flag. + * behavior already. When holding the exclusive lock for a write that + * needs partial block zeroing or is extending the file, we must wait + * for the I/O to complete synchronously: + * + * - needs_zeroing: drain in-flight DIO whose end_io could race with + * our partial block zeroing, and force synchronous completion so we + * don't leave in-flight zeroing bios for the next writer to drain. + * + * - extend: the caller must update i_disksize after I/O completion, + * which requires the data to be on disk first. */ - if (!*ilock_shared && (unaligned_io || *extend)) { + if (!*ilock_shared && (needs_zeroing || *extend)) { if (iocb->ki_flags & IOCB_NOWAIT) { ret = -EAGAIN; goto out; } - if (unaligned_io && (!overwrite || unwritten)) + if (needs_zeroing) inode_dio_wait(inode); *dio_flags = IOMAP_DIO_FORCE_WAIT; }