From b27d088e586b2464c7eb95dc1b493bea4c373ac7 Mon Sep 17 00:00:00 2001 From: Piotr Gorski Date: Fri, 6 Oct 2023 20:40:45 +0200 Subject: [PATCH] spadfs-6.6: merge v1.0.18 Signed-off-by: Piotr Gorski --- fs/Kconfig | 1 + fs/Makefile | 1 + fs/spadfs/Kconfig | 2 + fs/spadfs/Makefile | 4 + fs/spadfs/alloc.c | 2408 ++++++++++++++++++++++++++++++++++++++ fs/spadfs/allocmem.c | 217 ++++ fs/spadfs/buffer.c | 350 ++++++ fs/spadfs/bufstruc.c | 161 +++ fs/spadfs/dir.c | 2188 ++++++++++++++++++++++++++++++++++ fs/spadfs/dir.h | 167 +++ fs/spadfs/endian.h | 7 + fs/spadfs/file.c | 1904 ++++++++++++++++++++++++++++++ fs/spadfs/inode.c | 814 +++++++++++++ fs/spadfs/ioctl.c | 37 + fs/spadfs/link.c | 127 ++ fs/spadfs/linux-compat.h | 290 +++++ fs/spadfs/name.c | 109 ++ fs/spadfs/namei.c | 1204 +++++++++++++++++++ fs/spadfs/spadfs.h | 940 +++++++++++++++ fs/spadfs/struct.h | 965 +++++++++++++++ fs/spadfs/super.c | 1741 +++++++++++++++++++++++++++ fs/spadfs/xattr.c | 352 ++++++ 22 files changed, 13989 insertions(+) create mode 100644 fs/spadfs/Kconfig create mode 100644 fs/spadfs/Makefile create mode 100644 fs/spadfs/alloc.c create mode 100644 fs/spadfs/allocmem.c create mode 100644 fs/spadfs/buffer.c create mode 100644 fs/spadfs/bufstruc.c create mode 100644 fs/spadfs/dir.c create mode 100644 fs/spadfs/dir.h create mode 100644 fs/spadfs/endian.h create mode 100644 fs/spadfs/file.c create mode 100644 fs/spadfs/inode.c create mode 100644 fs/spadfs/ioctl.c create mode 100644 fs/spadfs/link.c create mode 100644 fs/spadfs/linux-compat.h create mode 100644 fs/spadfs/name.c create mode 100644 fs/spadfs/namei.c create mode 100644 fs/spadfs/spadfs.h create mode 100644 fs/spadfs/struct.h create mode 100644 fs/spadfs/super.c create mode 100644 fs/spadfs/xattr.c diff --git a/fs/Kconfig b/fs/Kconfig index aa7e03cc1..adec2739e 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -49,6 +49,7 @@ source "fs/btrfs/Kconfig" source "fs/nilfs2/Kconfig" source "fs/f2fs/Kconfig" source "fs/zonefs/Kconfig" +source "fs/spadfs/Kconfig" endif # BLOCK diff --git a/fs/Makefile b/fs/Makefile index f9541f40b..8a9d8a351 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -114,6 +114,7 @@ obj-$(CONFIG_XFS_FS) += xfs/ obj-$(CONFIG_9P_FS) += 9p/ obj-$(CONFIG_AFS_FS) += afs/ obj-$(CONFIG_NILFS2_FS) += nilfs2/ +obj-$(CONFIG_SPADFS_FS) += spadfs/ obj-$(CONFIG_BEFS_FS) += befs/ obj-y += hostfs/ obj-$(CONFIG_CACHEFILES) += cachefiles/ diff --git a/fs/spadfs/Kconfig b/fs/spadfs/Kconfig new file mode 100644 index 000000000..5fb2a0382 --- /dev/null +++ b/fs/spadfs/Kconfig @@ -0,0 +1,2 @@ +config SPADFS_FS + tristate "SPADFS file system support" diff --git a/fs/spadfs/Makefile b/fs/spadfs/Makefile new file mode 100644 index 000000000..d55536e3c --- /dev/null +++ b/fs/spadfs/Makefile @@ -0,0 +1,4 @@ +SPADFS_NAME = spadfs +CFLAGS_super.o := -DSPADFS_NAME=$(SPADFS_NAME) +obj-$(CONFIG_SPADFS_FS) += $(SPADFS_NAME).o +$(SPADFS_NAME)-y := alloc.o allocmem.o buffer.o bufstruc.o dir.o file.o inode.o ioctl.o link.o name.o namei.o super.o xattr.o diff --git a/fs/spadfs/alloc.c b/fs/spadfs/alloc.c new file mode 100644 index 000000000..c8bc6178e --- /dev/null +++ b/fs/spadfs/alloc.c @@ -0,0 +1,2408 @@ +#include "spadfs.h" + +#define MAX_CACHED_APAGE_BUFFERS 32 + +#define DECREASE_FREESPACE 0 +#define INCREASE_FREESPACE 1 + +/* Helper function for group_action */ + +static int group_action_1(SPADFS *fs, unsigned group, unsigned len, int action) +{ + if (likely(action == INCREASE_FREESPACE)) { + if (unlikely(fs->group_info[group].freespace + len < len) || + unlikely(fs->group_info[group].freespace + len > fs->group_mask + 1)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "allocation count for group %u overflow: %Lx, %Lx, %x", + group, + (unsigned long long)fs->group_info[group].freespace, + (unsigned long long)fs->group_info[group].zone->freespace, + len); + return -EFSERROR; + } + fs->group_info[group].freespace += len; + fs->group_info[group].zone->freespace += len; + } else { + if (unlikely(fs->group_info[group].freespace < len) || + unlikely(fs->group_info[group].zone->freespace < len)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "allocation count for group %u underflow: %Lx, %Lx, %x", + group, + (unsigned long long)fs->group_info[group].freespace, + (unsigned long long)fs->group_info[group].zone->freespace, + len); + return -EFSERROR; + } + fs->group_info[group].freespace -= len; + fs->group_info[group].zone->freespace -= len; + } + return 0; +} + +/* + * Action is either DECREASE_FREESPACE or INCREASE_FREESPACE. + * This function will update in-memory group statistics (and handle correctly + * situations like alloc or free crossing multiple groups). + */ + +static int group_action(SPADFS *fs, sector_t start, unsigned len, int action) +{ + unsigned start_group, end_group; + start_group = start >> fs->sectors_per_group_bits; + end_group = (start + (len - 1)) >> fs->sectors_per_group_bits; + if (unlikely(end_group >= fs->n_groups)) { +range_error: + spadfs_error(fs, TXFLAGS_FS_ERROR, + "invalid range in group action (%Lx,%x)", + (unsigned long long)start, len); + return -EFSERROR; + } + if (unlikely(start_group > end_group)) { + goto range_error; + } + if (likely(start_group == end_group)) { + return group_action_1(fs, start_group, len, action); + } else { + int r = group_action_1(fs, start_group, + fs->group_mask + 1 - (start & fs->group_mask), + action); + if (unlikely(r)) + return r; + for (start_group++; start_group < end_group; start_group++) { + r = group_action_1(fs, start_group, + fs->group_mask + 1, action); + if (unlikely(r)) + return r; + } + r = group_action_1(fs, start_group, + ((start + len - 1) & fs->group_mask) + 1, action); + return r; + } +} + +static void freespace_decrease(SPADFS *fs, sector_t sector, unsigned n_sectors) +{ + fs->freespace -= n_sectors; + if (unlikely(fs->max_allocation > fs->freespace)) + fs->max_allocation = fs->freespace; + group_action(fs, sector, n_sectors, DECREASE_FREESPACE); +} + +static void freespace_increase(SPADFS *fs, sector_t sector, unsigned n_sectors) +{ + if (unlikely(!fs->max_allocation)) + fs->max_allocation = 1U << fs->sectors_per_disk_block_bits; + fs->freespace += n_sectors; + group_action(fs, sector, n_sectors, INCREASE_FREESPACE); +} + +static void spadfs_discard_reservation_unlocked(SPADFS *fs, struct alloc_reservation *res) +{ + res->len = 0; + rb_erase(&res->rb_node, &fs->alloc_reservations); +} + +void spadfs_discard_reservation(SPADFS *fs, struct alloc_reservation *res) +{ + if (res->len) { + mutex_lock(&fs->alloc_lock); + if (likely(res->len != 0)) + spadfs_discard_reservation_unlocked(fs, res); + mutex_unlock(&fs->alloc_lock); + } +} + +static int spadfs_discard_all_reservations(SPADFS *fs) +{ + struct rb_node *p; + if (!(p = fs->alloc_reservations.rb_node)) + return 0; + do { +#define res rb_entry(p, struct alloc_reservation, rb_node) + res->len = 0; + rb_erase(&res->rb_node, &fs->alloc_reservations); +#undef res + } while ((p = fs->alloc_reservations.rb_node)); + return 1; +} + +static int spadfs_check_reservations(SPADFS *fs, sector_t start, unsigned n_sec, sector_t *new_free) +{ + struct rb_node *p = fs->alloc_reservations.rb_node; + while (p) { +#define res rb_entry(p, struct alloc_reservation, rb_node) + if (start + n_sec <= res->start) { + p = res->rb_node.rb_left; + } else if (likely(start >= res->start + res->len)) { + p = res->rb_node.rb_right; + } else { + *new_free = res->start + res->len; + return 1; + } +#undef res + } + return 0; +} + +static void spadfs_add_reservation(SPADFS *fs, struct alloc_reservation *new) +{ + struct rb_node **p = &fs->alloc_reservations.rb_node; + struct rb_node *parent = NULL; + while (*p) { + parent = *p; +#define res rb_entry(parent, struct alloc_reservation, rb_node) + if (new->start + new->len <= res->start) { + p = &res->rb_node.rb_left; + } else if (likely(new->start >= res->start + res->len)) { + p = &res->rb_node.rb_right; + } else { + printk(KERN_EMERG "spadfs: new reservation %Lx,%Lx overlaps with %Lx,%Lx\n", + (unsigned long long)new->start, + (unsigned long long)(new->start + new->len), + (unsigned long long)res->start, + (unsigned long long)(res->start + res->len)); + BUG(); + } +#undef res + } + rb_link_node(&new->rb_node, parent, p); + rb_insert_color(&new->rb_node, &fs->alloc_reservations); +} + +/* + * Convert pointer to mapped apage to index of an apage on filesystem + * --- for error prints + */ + +static int map_2_apage_n(SPADFS *fs, APAGE_MAP *map) +{ + unsigned i, j; + if (map == fs->tmp_map) + return -1; + for (i = 0; i < fs->n_apages; i++) + for (j = 0; j < 2; j++) + if (fs->apage_info[i].mapping[j].map == map) + return i; + panic("spadfs: map_2_apage_n: invalid pointer %p", map); +} + +/* Get 16-byte entry in mapped apage */ + +static struct aentry *get_aentry(SPADFS *fs, APAGE_MAP *map, unsigned off) +{ + if (unlikely(off & (sizeof(struct aentry) - 1))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "unaligned apage entry %u, apage %d", + off, map_2_apage_n(fs, map)); + return ERR_PTR(-EFSERROR); + } + if (unlikely(off >= fs->apage_size)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "apage entry %u too big, apage %d", + off, map_2_apage_n(fs, map)); + return ERR_PTR(-EFSERROR); + } + return (struct aentry *)((char *)map[off >> (9 + fs->sectors_per_buffer_bits)].entry + (off & ((512U << fs->sectors_per_buffer_bits) - 1))); +} + +/* Get 16-byte entry in mapped apage assuming that the index is valid */ + +static struct aentry *get_aentry_valid(SPADFS *fs, APAGE_MAP *map, unsigned off) +{ +#ifdef CONFIG_DEBUG_LIST + if (unlikely(off & (sizeof(struct aentry) - 1)) + || unlikely(off >= fs->apage_size)) { + panic("spadfs: get_aentry_valid: invalid offset %u", off); + } +#endif + return (struct aentry *)((char *)map[off >> (9 + fs->sectors_per_buffer_bits)].entry + (off & ((512U << fs->sectors_per_buffer_bits) - 1))); +} + +/* Get first 16-byte entry of an apage --- its head */ + +static struct apage_head *get_head(APAGE_MAP *map) +{ + return (struct apage_head *)map[0].entry; +} + +static void internal_free_apage_buffers(SPADFS *fs, struct apage_mapping *mapping) +{ + unsigned n; + APAGE_MAP *map; + + list_del(&mapping->list); + BUG_ON(!fs->cached_apage_buffers); + fs->cached_apage_buffers--; + + n = fs->n_apage_mappings; + map = mapping->map; + while (n--) { + map->entry = NULL; + if (likely(map->bh != NULL)) + spadfs_brelse(fs, map->bh); + map++; + } +} + +static void spadfs_prune_cached_apage_buffer(SPADFS *fs) +{ + struct apage_mapping *mapping; + + BUG_ON(list_empty(&fs->apage_lru)); + + mapping = list_entry(fs->apage_lru.next, struct apage_mapping, list); + + internal_free_apage_buffers(fs, mapping); +} + +void spadfs_prune_cached_apage_buffers(SPADFS *fs) +{ + BUG_ON(fs->mapped_apage_buffers); + while (!list_empty(&fs->apage_lru)) { + spadfs_prune_cached_apage_buffer(fs); + } + BUG_ON(fs->cached_apage_buffers); + BUG_ON(!list_empty(&fs->apage_lru)); +} + +static void free_apage_buffers(SPADFS *fs, APAGE_MAP *map) +{ + BUG_ON(!fs->mapped_apage_buffers); + fs->mapped_apage_buffers--; + if (!fs->mapped_apage_buffers) { + while (unlikely(fs->cached_apage_buffers > MAX_CACHED_APAGE_BUFFERS)) + spadfs_prune_cached_apage_buffer(fs); + } +} + +static noinline int internal_read_apage_buffers(SPADFS *fs, struct apage_mapping *mapping, sector_t sec) +{ + unsigned i; + fs->cached_apage_buffers++; + list_add_tail(&mapping->list, &fs->apage_lru); + mapping->map[0].entry = spadfs_read_sector(fs, sec, &mapping->map[0].bh, fs->n_apage_mappings << fs->sectors_per_buffer_bits, "internal_read_apage_buffers 1"); + if (unlikely(IS_ERR(mapping->map[0].entry))) { + i = 0; + goto error; + } + for (i = 1; i < fs->n_apage_mappings; i++) { + sec += 1U << fs->sectors_per_buffer_bits; + mapping->map[i].entry = spadfs_read_sector(fs, sec, &mapping->map[i].bh, 0, "internal_read_apage_buffers 2"); + if (unlikely(IS_ERR(mapping->map[i].entry))) { + int r; +error: + r = PTR_ERR(mapping->map[i].entry); + internal_free_apage_buffers(fs, mapping); + fs->mapped_apage_buffers--; + return r; + } + } + return 0; +} + +static int read_apage_buffers(SPADFS *fs, struct apage_mapping *mapping, sector_t sec) +{ + fs->mapped_apage_buffers++; + if (likely(mapping->map[0].entry != NULL)) { + list_del(&mapping->list); + list_add_tail(&mapping->list, &fs->apage_lru); + return 0; + } + return internal_read_apage_buffers(fs, mapping, sec); +} + +/* Copy one mapped apage to another */ + +static void copy_apage(SPADFS *fs, APAGE_MAP *dst, APAGE_MAP *src) +{ + unsigned i; + unsigned n = fs->n_apage_mappings; + unsigned size = 512U << fs->sectors_per_buffer_bits; + const unsigned offset = sizeof(struct apage_head) - sizeof(struct apage_subhead); + if (unlikely(fs->apage_size < size)) + size = fs->apage_size; + memcpy((char *)dst[0].entry + offset, (char *)src[0].entry + offset, size - offset); + for (i = 1; i < n; i++) + memcpy(dst[i].entry, src[i].entry, size); +} + +static void mark_apage_dirty(SPADFS *fs, APAGE_MAP *map) +{ + unsigned i; + for (i = 0; i < fs->n_apage_mappings; i++) + mark_buffer_dirty(map[i].bh); +} + +#define invalid_flags(fs, flags) \ + unlikely(((flags) & (APAGE_SIZE_BITS | APAGE_BLOCKSIZE_BITS)) !=\ + ((((fs)->sectors_per_page_bits - 1) << APAGE_SIZE_BITS_SHIFT) | \ + ((fs)->sectors_per_disk_block_bits << APAGE_BLOCKSIZE_BITS_SHIFT))) + +/* + * Map existing apage apage + * ap is index of an apage + * flags is one of + * MAP_READ --- map apage for read + * MAP_NEW --- map yet unused apage + * MAP_WRITE --- map apage for write (i.e. fixup cc/txc and copy its + * content if its the first map since last sync) + * MAP_ALLOC --- like MAP_WRITE but skip completely full apages + * (an optimization) + */ + +#define MAP_READ 0 +#define MAP_NEW 1 +#define MAP_WRITE 2 +#define MAP_ALLOC 3 + +static APAGE_MAP *map_apage(SPADFS *fs, unsigned ap, int flags, + APAGE_MAP **other) +{ + struct apage_info *info; + unsigned idx; + int r; + sector_t sec; + struct apage_head *a; + struct apage_head *aa; + sec = SPAD2CPU64_LV(&fs->apage_index[ap].apage); + if (unlikely((unsigned)sec & ((1U << fs->sectors_per_page_bits) - 1))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "misaligned apage %u at block %Lx", + ap, (unsigned long long)sec); + return ERR_PTR(-EFSERROR); + } + info = &fs->apage_info[ap]; + if (unlikely(r = read_apage_buffers(fs, &info->mapping[0], sec))) { + return ERR_PTR(r); + } + if (unlikely(r = read_apage_buffers(fs, &info->mapping[1], + sec + (1U << (fs->sectors_per_page_bits - 1))))) { + free_apage_buffers(fs, info->mapping[0].map); + return ERR_PTR(r); + } + + a = get_head(info->mapping[0].map); + if (unlikely(a->magic != CPU2SPAD16_CONST(APAGE_MAGIC))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, "bad magic on apage %Lx", + (unsigned long long)sec); + r = -EFSERROR; + goto ret_err; + } + + if (unlikely(flags == MAP_NEW)) { + start_atomic_buffer_modify(fs, info->mapping[0].map[0].bh); + CC_SET_CURRENT_INVALID(fs, &a->cc, &a->txc); + end_atomic_buffer_modify(fs, info->mapping[0].map[0].bh); + } + + idx = !CC_VALID(fs, &a->cc, &a->txc); + + if (unlikely(flags == MAP_NEW)) { + goto read_ok; + } + + if (invalid_flags(fs, get_head(info->mapping[idx].map)->s.u.l.flags)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "apage %Lx has invalid flags %02x", + (unsigned long long)sec, + get_head(info->mapping[idx].map)->s.u.l.flags); + r = -EFSERROR; + goto ret_err; + } + + if (unlikely(flags == MAP_READ)) { +read_ok: + if (likely(other != NULL)) + *other = info->mapping[idx ^ 1].map; + else + free_apage_buffers(fs, info->mapping[idx ^ 1].map); + + return info->mapping[idx].map; + } + if (likely(flags == MAP_ALLOC)) { + aa = get_head(info->mapping[idx].map); + if (unlikely(!(aa->s.u.l.flags & APAGE_BITMAP)) && + unlikely(aa->s.u.l.last == CPU2SPAD16_CONST(0))) { + r = -ENOSPC; + goto ret_err; + } + } + if (likely(CC_CURRENT(fs, &a->cc, &a->txc))) { + if (invalid_flags(fs, get_head(info->mapping[idx ^ 1].map)->s.u.l.flags)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "apage %Lx has invalid flags %02x in inactive part", + (unsigned long long)sec, + get_head(info->mapping[idx ^ 1].map)->s.u.l.flags); + r = -EFSERROR; + goto ret_err; + } + goto read_ok; + } + start_atomic_buffer_modify(fs, info->mapping[0].map[0].bh); + CC_SET_CURRENT(fs, &a->cc, &a->txc); + end_atomic_buffer_modify(fs, info->mapping[0].map[0].bh); + + copy_apage(fs, info->mapping[idx ^ 1].map, info->mapping[idx].map); + mark_apage_dirty(fs, info->mapping[idx ^ 1].map); + idx ^= 1; + goto read_ok; + +ret_err: + free_apage_buffers(fs, info->mapping[0].map); + free_apage_buffers(fs, info->mapping[1].map); + return ERR_PTR(r); +} + +/* + * Unmap existing apage --- a companion to map_apage + * If the apage has been modified, "modified" argument must be nonzero + */ + +static void unmap_apage(SPADFS *fs, APAGE_MAP *map, int modified) +{ + BUG_ON(!atomic_read(&map[0].bh->b_count)); + if (modified) { + get_head(map)->s.u.l.flags &= ~APAGE_CHECKSUM_VALID; + mark_apage_dirty(fs, map); + } + free_apage_buffers(fs, map); +} + +/* + * Create apage that has all entries free (i.e. it doesn't specify any free + * space yet) + */ + +__cold static void make_apage(SPADFS *fs, APAGE_MAP *map) +{ + unsigned i; + struct apage_head *head; + struct aentry *ae; + head = get_head(map); + head->s.u.l.flags = + ((fs->sectors_per_page_bits - 1) << APAGE_SIZE_BITS_SHIFT) | + (fs->sectors_per_disk_block_bits << APAGE_BLOCKSIZE_BITS_SHIFT); + head->s.u.l.checksum = 0; + CPU2SPAD16_LV(&head->s.u.l.freelist, sizeof(struct aentry)); + CPU2SPAD16_LV(&head->s.u.l.last, 0); + CPU2SPAD16_LV(&head->s.u.l.first, 0); + i = sizeof(struct aentry); + do { + ae = get_aentry_valid(fs, map, i); + CPU2SPAD64_LV(&ae->start, 0); + CPU2SPAD32_LV(&ae->len, 0); + CPU2SPAD16_LV(&ae->prev, 0); + i += sizeof(struct aentry); + CPU2SPAD16_LV(&ae->next, i); + } while (i < fs->apage_size); + CPU2SPAD16_LV(&ae->next, 0); +} + +/* Create bitmap apage that has all bits marked as "allocated" */ + +__cold static void make_apage_bitmap(SPADFS *fs, APAGE_MAP *map, sector_t start) +{ + unsigned i; + struct apage_head *head; + struct aentry *ae; + head = get_head(map); + head->s.u.b.flags = + ((fs->sectors_per_page_bits - 1) << APAGE_SIZE_BITS_SHIFT) | + (fs->sectors_per_disk_block_bits << APAGE_BLOCKSIZE_BITS_SHIFT) | + APAGE_BITMAP; + head->s.u.b.checksum = 0; + head->s.u.b.start1 = MAKE_PART_1(start); + head->s.u.b.start0 = MAKE_PART_0(start); + i = sizeof(struct aentry); + do { + ae = get_aentry_valid(fs, map, i); + memset(ae, 0xff, sizeof(struct aentry)); + i += sizeof(struct aentry); + } while (i < fs->apage_size); +} + +/* + * Helper for spadfs_count_free_space + * Count free space in one apage + */ + +static int get_map_stats(SPADFS *fs, APAGE_MAP *map, sector_t *freespace) +{ + struct apage_head *head = get_head(map); + if (likely(!(head->s.u.l.flags & APAGE_BITMAP))) { + unsigned p = SPAD2CPU16_LV(&head->s.u.l.first); + unsigned max_loop = (512 / sizeof(struct aentry) / 2) << + fs->sectors_per_page_bits; + while (p) { + struct aentry *aentry = get_aentry(fs, map, p); + if (unlikely(IS_ERR(aentry))) + return PTR_ERR(aentry); + if (unlikely(group_action(fs, + SPAD2CPU64_LV(&aentry->start), + SPAD2CPU32_LV(&aentry->len), + INCREASE_FREESPACE))) + return -EFSERROR; + *freespace += SPAD2CPU32_LV(&aentry->len); + p = SPAD2CPU16_LV(&aentry->next); + if (unlikely(!--max_loop)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "infinite loop in apage %d in get_map_stats", + map_2_apage_n(fs, map)); + return -EFSERROR; + } + } + return 0; + } else { + unsigned p; + sector_t bst = MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1); + for (p = sizeof(struct aentry); p < fs->apage_size; + p += sizeof(struct aentry)) { + unsigned i; + u8 *bm = (u8 *)get_aentry(fs, map, p); + if (unlikely(IS_ERR(bm))) + return PTR_ERR(bm); + for (i = 0; i < sizeof(struct aentry) * 8; i++) { + if (BITMAP_TEST_32_FULL(bm - sizeof(struct apage_head), i)) { + i += 31; + continue; + } + if (!BITMAP_TEST(bm - sizeof(struct apage_head), i)) { + *freespace += 1U << fs->sectors_per_disk_block_bits; + if (unlikely(group_action(fs, + bst + ((i + (p - sizeof(struct aentry)) * 8) << fs->sectors_per_disk_block_bits), + 1U << fs->sectors_per_disk_block_bits, + INCREASE_FREESPACE))) + return -EFSERROR; + } + } + } + return 0; + } +} + +/* Counts free space during mount */ + +int spadfs_count_free_space(SPADFS *fs) +{ + unsigned i; + sector_t freespace = 0; + for (i = 0; i < fs->n_active_apages; i++) { + int r; + APAGE_MAP *map; + if (unlikely(IS_ERR(map = map_apage(fs, i, MAP_READ, NULL)))) + return PTR_ERR(map); + r = get_map_stats(fs, map, &freespace); + unmap_apage(fs, map, 0); + if (unlikely(r)) return r; + } + fs->freespace = freespace; + fs->max_allocation = freespace; + if (unlikely(freespace != fs->max_allocation)) + fs->max_allocation = -(1U << fs->sectors_per_disk_block_bits); + + freespace = 0; + for (i = 0; i < fs->n_groups; i++) { + freespace += fs->group_info[i].freespace; + } + if (unlikely(freespace != fs->freespace)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "free space in groups miscounted, %Lx != %Lx", + (unsigned long long)freespace, + (unsigned long long)fs->freespace); + return -EFSERROR; + } + + freespace = 0; + for (i = 0; i < 3; i++) { + freespace += fs->zones[i].freespace; + } + if (unlikely(freespace != fs->freespace)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "free space in zones miscounted, %Lx != %Lx", + (unsigned long long)freespace, + (unsigned long long)fs->freespace); + return -EFSERROR; + } + + return 0; +} + +/* + * Find the last apage entry mapping area before the specified block + * Uses algorithm with average O(sqrt(n)) complexity: + * Scan any sqrt(n) blocks (the blocks are selected deliberately in + * 64-byte chunks (4 entries, each 16 bytes) to minimize cache + * pollution) + * From scanned blocks, select the one with highest start that is lower + * than "before" + * From the selected block, scan forward using double-linked list pointers + * (on average we scan sqrt(n) apage entries in this pass) + */ + +static int find_block_before(SPADFS *fs, APAGE_MAP *map, sector_t before) +{ + unsigned i; + unsigned rlimit; + sector_t xst = 0; + unsigned st = 0, nx; + struct aentry *sta = (struct aentry *)get_head(map), *nxa; + for (i = 4 * sizeof(struct aentry); + i <= fs->apage_size - 4 * sizeof(struct aentry); + i += 124 * sizeof(struct aentry)) { + unsigned j; + for (j = 0; j < 4; j++) { + struct aentry *a = get_aentry_valid(fs, map, i); + if (unlikely(IS_ERR(a))) + return PTR_ERR(a); + if (a->len != CPU2SPAD32_CONST(0) && + SPAD2CPU64_LV(&a->start) <= before && + unlikely(SPAD2CPU64_LV(&a->start) >= xst)) { + xst = SPAD2CPU64_LV(&a->start); + st = i; + sta = a; + } + i += sizeof(struct aentry); + } + } + rlimit = fs->apage_size; + do { + nx = SPAD2CPU16_LV(&sta->next); + nxa = get_aentry(fs, map, nx); + if (unlikely(IS_ERR(nxa))) + return PTR_ERR(nxa); + if (unlikely(!nx) || + SPAD2CPU64_LV(&nxa->start) > before) + return st; + st = nx; + sta = nxa; + } while (likely(rlimit -= sizeof(struct aentry))); + spadfs_error(fs, TXFLAGS_FS_ERROR, + "infinite loop in apage %d in find_block_before", + map_2_apage_n(fs, map)); + return -EFSERROR; +} + +/* Simple operations with bits in bitmaps */ + +static int bmp_test(SPADFS *fs, APAGE_MAP *map, unsigned off) +{ + u32 *p = (u32 *)get_aentry_valid(fs, map, + (1 + off / (sizeof(struct aentry) * 8)) * + sizeof(struct aentry)); + p += (off % (sizeof(struct aentry) * 8)) >> 5; + return (SPAD2CPU32_LV(p) >> (off & 31)) & 1; +} + +static void bmp_set(SPADFS *fs, APAGE_MAP *map, unsigned off) +{ + u32 *p = (u32 *)get_aentry_valid(fs, map, + (1 + off / (sizeof(struct aentry) * 8)) * + sizeof(struct aentry)); + p += (off % (sizeof(struct aentry) * 8)) >> 5; + CPU2SPAD32_LV(p, SPAD2CPU32_LV(p) | (1U << (off & 31))); +} + +static void bmp_clear(SPADFS *fs, APAGE_MAP *map, unsigned off) +{ + u32 *p = (u32 *)get_aentry_valid(fs, map, + (1 + off / (sizeof(struct aentry) * 8)) * + sizeof(struct aentry)); + p += (off % (sizeof(struct aentry) * 8)) >> 5; + CPU2SPAD32_LV(p, SPAD2CPU32_LV(p) & ~(1U << (off & 31))); +} + +static int bmp_test_32_full(SPADFS *fs, APAGE_MAP *map, unsigned off) +{ + u32 *p = (u32 *)get_aentry_valid(fs, map, + (1 + off / (sizeof(struct aentry) * 8)) * + sizeof(struct aentry)); + p += (off % (sizeof(struct aentry) * 8)) >> 5; + return *p == CPU2SPAD32(0xffffffffu); +} + +/* Helper for check_other_map */ + +static int check_other_map_bmp(SPADFS *fs, APAGE_MAP *map, sector_t off, + unsigned n_sec, sector_t *next_free) +{ + struct apage_head *head = get_head(map); + unsigned Xboff = BITMAP_OFFSET(head, off); + unsigned Xn_sec = BITMAP_LEN(head, n_sec); + if (unlikely(Xboff + Xn_sec <= Xboff) || + unlikely(Xboff + Xn_sec > BITMAP_SIZE(fs->apage_size))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "bitmap error, apage %d, Xboff %x, off %Lx, n_sec %x", + map_2_apage_n(fs, map), Xboff, (unsigned long long)off, + n_sec); + return -EFSERROR; + } + do { + if (unlikely(bmp_test(fs, map, Xboff))) { + while (++Xboff < BITMAP_SIZE(fs->apage_size)) + if (!bmp_test(fs, map, Xboff)) { + *next_free = MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1) + + (Xboff << fs->sectors_per_disk_block_bits); + return -EBUSY; + } + return -EBUSY; + } + Xboff++; + } while (--Xn_sec); + return 0; +} + +/* + * When we are allocating space, we must make sure that we don't allocate space + * that is free but that would be valid in case of crash --- this functions + * checks it + */ + +static int check_other_map(SPADFS *fs, APAGE_MAP *map, sector_t off, + unsigned n_sec, sector_t *next_free) +{ + if (unlikely(!map)) + return 0; + *next_free = 0; + if (likely(!(get_head(map)->s.u.l.flags & APAGE_BITMAP))) { + int preblk, postblk; + struct aentry *e; + preblk = find_block_before(fs, map, off); + if (unlikely(preblk < 0)) + return preblk; + e = get_aentry_valid(fs, map, preblk); + if (likely(preblk != 0)) { + if (off + n_sec <= + SPAD2CPU64_LV(&e->start) + SPAD2CPU32_LV(&e->len)) + return 0; + } + postblk = SPAD2CPU16_LV(&e->next); + if (likely(postblk != 0)) { + e = get_aentry(fs, map, postblk); + if (unlikely(IS_ERR(e))) + return PTR_ERR(e); + *next_free = SPAD2CPU64_LV(&e->start); + } + return -EBUSY; + } else + return check_other_map_bmp(fs, map, off, n_sec, next_free); +} + +static int check_conflict(SPADFS *fs, APAGE_MAP *other, sector_t start, unsigned n_sec, sector_t *new_free) +{ + if ((other && unlikely(check_other_map(fs, other, start, n_sec, new_free))) || + unlikely(spadfs_allocmem_find(fs, start, n_sec, new_free)) || + unlikely(spadfs_check_reservations(fs, start, n_sec, new_free))) + return 1; + return 0; +} + +#ifdef SPADFS_RESURRECT +static int check_resurrect_map(SPADFS *fs, APAGE_MAP *map, sector_t off, + unsigned n_sec) +{ + sector_t next_free_sink; + int r; + + r = check_other_map(fs, map, off, n_sec, &next_free_sink); + if (likely(r == -EBUSY)) + return 0; + + if (!r) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "no shadow when resurrecting blocks (%Lx,%x)", + (unsigned long long)off, + n_sec); + r = -EFSERROR; + } + + return r; +} +#endif + +/* Helper for alloc_blocks_in_apage --- do allocation in case it is bitmap */ + +static int alloc_blocks_in_apage_bmp(SPADFS *fs, APAGE_MAP *map, + APAGE_MAP *other, struct alloc *al) +{ + struct apage_head *head = get_head(map); + unsigned shift = fs->sectors_per_disk_block_bits; + unsigned Xbmax = BITMAP_SIZE(fs->apage_size); + unsigned Xboff; + sector_t bst = MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1); + sector_t bottom, top; + bottom = al->sector; + if (bottom < bst) + bottom = bst; + top = al->top; + if (likely(top > bst + (Xbmax << shift))) + top = bst + (Xbmax << shift); + if (unlikely(bottom >= top)) + return -ENOSPC; + Xboff = (bottom - bst) >> shift; + if (top - bst < (Xbmax << shift)) + Xbmax = (top - bst) >> shift; + while (Xboff < Xbmax) { + if (likely(bmp_test_32_full(fs, map, Xboff))) { + Xboff = (Xboff + 32) & ~31; + continue; + } +test_bit: + if (unlikely(!bmp_test(fs, map, Xboff))) { + sector_t start; + sector_t new_bot; + unsigned Xc; + unsigned limit; + if (unlikely(al->flags & ALLOC_METADATA)) { + if ((((unsigned)bst + (Xboff << shift)) & + ALLOC_MASK(al->flags)) + al->n_sectors > + ALLOC_MASK(al->flags) + 1) + goto align; + } else if (((unsigned)bst + (Xboff << shift)) & + ALLOC_MASK(al->flags)) { + unsigned Xplus; +align: + Xplus = (ALLOC_MASK(al->flags) + 1 - + (((unsigned)bst + (Xboff << shift)) & + ALLOC_MASK(al->flags))) >> shift; + if (unlikely(!Xplus)) + Xplus = 1; + Xboff += Xplus; + continue; + } + limit = (al->n_sectors + al->extra_sectors) >> shift; + for (Xc = 1; Xc < limit; Xc++) + if (unlikely(Xboff + Xc >= Xbmax) || + bmp_test(fs, map, Xboff + Xc)) { + Xboff += Xc + 1; + goto cont; + } + start = bst + (Xboff << shift); +#ifdef SPADFS_RESURRECT + if (unlikely(al->flags & ALLOC_RESURRECT)) { + int r = check_resurrect_map(fs, other, start, + al->n_sectors); + if (unlikely(r)) + return r; + } else +#endif + if (unlikely(check_conflict(fs, other, start, al->n_sectors + al->extra_sectors, &new_bot))) { + if (likely(new_bot > bst + (Xboff << shift)) && + likely(new_bot < bst + (Xbmax << shift))) { + Xboff = (new_bot - bst + + ((1U << shift) - 1)) >> shift; + continue; + } + return -ENOSPC; + } + al->sector = start; + limit = al->n_sectors >> shift; + for (Xc = 0; Xc < limit; Xc++) + bmp_set(fs, map, Xboff + Xc); + return 0; + } + Xboff++; + if (likely(Xboff < Xbmax) && likely(Xboff & 31)) + goto test_bit; + cont:; + } + return -ENOSPC; +} + +/* + * Delete one aentry from double linked list and add it to single linked + * freelist + */ + +static void delete_block(SPADFS *fs, APAGE_MAP *map, struct aentry *e) +{ + struct aentry *pre, *post; + struct apage_head *head; + u16 nblk; + pre = get_aentry(fs, map, SPAD2CPU16_LV(&e->prev)); + if (unlikely(IS_ERR(pre))) + return; + pre->next = e->next; + post = get_aentry(fs, map, SPAD2CPU16_LV(&e->next)); + if (unlikely(IS_ERR(post))) + return; + nblk = post->prev; + post->prev = e->prev; + CPU2SPAD64_LV(&e->start, 0); + CPU2SPAD32_LV(&e->len, 0); + CPU2SPAD16_LV(&e->prev, 0); + head = get_head(map); + e->next = head->s.u.l.freelist; + head->s.u.l.freelist = nblk; +} + +/* Alloc block in a given aentry */ + +static int alloc_block(SPADFS *fs, struct aentry *a, struct alloc *al, + APAGE_MAP *other) +{ + sector_t bottom, top, new_bot; + bottom = al->sector; + if (likely(bottom < SPAD2CPU64_LV(&a->start))) + bottom = SPAD2CPU64_LV(&a->start); + top = al->top; + if (likely(top > SPAD2CPU64_LV(&a->start) + SPAD2CPU32_LV(&a->len))) + top = SPAD2CPU64_LV(&a->start) + SPAD2CPU32_LV(&a->len); +new_bottom: + if (unlikely(al->flags & ALLOC_METADATA)) + if (likely(((unsigned)bottom & ALLOC_MASK(al->flags)) + + al->n_sectors <= ALLOC_MASK(al->flags) + 1)) + goto skip_pad; + bottom = (bottom + ALLOC_MASK(al->flags)) & + ~(sector_t)ALLOC_MASK(al->flags); +skip_pad: + if (unlikely(bottom + al->n_sectors + al->extra_sectors > top)) + return -ENOSPC; + +#ifdef SPADFS_RESURRECT + if (unlikely(al->flags & ALLOC_RESURRECT)) { + int r = check_resurrect_map(fs, other, bottom, al->n_sectors); + if (unlikely(r)) + return r; + } else +#endif + if (unlikely(check_conflict(fs, other, bottom, al->n_sectors + al->extra_sectors, &new_bot))) { + if (likely(new_bot > bottom)) { + bottom = new_bot; + goto new_bottom; + } + return -ENOSPC; + } + if (unlikely(bottom > SPAD2CPU64_LV(&a->start))) { + al->flags |= ALLOC_FREE_FROM; + al->top = SPAD2CPU64_LV(&a->start); + } + CPU2SPAD32_LV(&a->len, SPAD2CPU32_LV(&a->len) - + (((u32)bottom - (u32)SPAD2CPU64_LV(&a->start)) + al->n_sectors)); + CPU2SPAD64_LV(&a->start, bottom + al->n_sectors); + al->sector = bottom; + return 0; +} + +/* Alloc block run in a given apage */ + +static int alloc_blocks_in_apage(SPADFS *fs, APAGE_MAP *map, APAGE_MAP *other, + struct alloc *al) +{ + struct apage_head *head = get_head(map); + int r; + + if (unlikely(IS_ERR(head))) + return PTR_ERR(head); + + if (likely(!(head->s.u.l.flags & APAGE_BITMAP))) { + struct aentry *e; + int n_cycles; + int blk = find_block_before(fs, map, al->sector); + if (unlikely(blk < 0)) + return blk; + e = get_aentry_valid(fs, map, blk); + n_cycles = fs->apage_size; +next_try: + if (!blk || + al->sector >= + SPAD2CPU64_LV(&e->start) + SPAD2CPU32_LV(&e->len)) { + blk = SPAD2CPU16_LV(&e->next); + /* blk is valid from find_block_before */ + if (unlikely(!blk)) + return -ENOSPC; + e = get_aentry_valid(fs, map, blk); + } + if (unlikely(SPAD2CPU64_LV(&e->start) >= al->top)) + return -ENOSPC; + if (SPAD2CPU32_LV(&e->len) >= al->n_sectors + al->extra_sectors) { + r = alloc_block(fs, e, al, other); + if (likely(!r)) { + if (unlikely(e->len == CPU2SPAD32(0))) + delete_block(fs, map, e); + return 0; + } + if (unlikely(r != -ENOSPC)) + return r; + } + + blk = SPAD2CPU16_LV(&e->next); + if (unlikely(!blk)) + return -ENOSPC; + e = get_aentry(fs, map, blk); + if (unlikely(blk < 0)) + return blk; + if (likely(n_cycles -= sizeof(struct aentry))) + goto next_try; + + spadfs_error(fs, TXFLAGS_FS_ERROR, + "infinite loop in apage %d in alloc_blocks_in_apage", + map_2_apage_n(fs, map)); + return -EFSERROR; + } else + return alloc_blocks_in_apage_bmp(fs, map, other, al); +} + +/* Map apage and alloc block run in it */ + +static int try_apage(SPADFS *fs, unsigned ap, struct alloc *al) +{ + int r; + APAGE_MAP *map, *other; + if (unlikely(IS_ERR(map = map_apage(fs, ap, MAP_ALLOC, &other)))) + return PTR_ERR(map); + r = alloc_blocks_in_apage(fs, map, other, al); + if (likely(!r)) { + unmap_apage(fs, map, 1); + unmap_apage(fs, other, 0); + return 0; + } + unmap_apage(fs, map, 0); + unmap_apage(fs, other, 0); + return r; +} + +/* Convert block number to an index of apage --- use binary search */ + +static int addr_2_apage(SPADFS *fs, sector_t o, char *msg) +{ + int a1, a2, a; + a2 = fs->n_active_apages - 1; + a1 = 0; +again: + a = (a1 + a2) >> 1; + if (o < (sector_t)SPAD2CPU64_LV(&fs->apage_index[a - 1].end_sector)) + a2 = a - 1; + else if (o >= (sector_t)SPAD2CPU64_LV(&fs->apage_index[a].end_sector)) + a1 = a + 1; + else + return a; + if (likely(a1 <= a2)) + goto again; + spadfs_error(fs, TXFLAGS_FS_ERROR, + "can't find apage for block %Lx, stuck on %d/%d, called from %s", + (unsigned long long)o, a1, a2, msg); + return -EFSERROR; +} + +/* Alloc block run in a range specified by al->sector and al->top */ + +static int alloc_blocks_in_range(SPADFS *fs, struct alloc *al) +{ + int ap, tap; + int r; + + if (unlikely(al->top > fs->size)) + al->top = fs->size; + if (unlikely(al->sector >= al->top)) + return -ENOSPC; + + ap = addr_2_apage(fs, al->sector, "alloc_blocks_in_range 1"); + if (unlikely(ap < 0)) + return ap; + if (likely(al->top <= SPAD2CPU64_LV(&fs->apage_index[ap].end_sector))) + tap = ap; + else { + tap = addr_2_apage(fs, al->top - 1, "alloc_blocks_in_range 2"); + if (unlikely(tap < 0)) + return tap; + } + +next_ap: + r = try_apage(fs, ap, al); + if (likely(!r)) + return 0; + if (unlikely(r != -ENOSPC)) + return r; + ap++; + if (unlikely(ap <= tap)) + goto next_ap; + return -ENOSPC; +} + +/* + * Get number of consecutive free blocks at given location + * "max" is the limit, don't scan past this, even if there are more free blocks + */ + +static unsigned get_blocklen_at(SPADFS *fs, sector_t block, unsigned max, sector_t *next) +{ + struct apage_head *head; + APAGE_MAP *map, *other = NULL; + unsigned ret; + int ap; + max &= -(1U << fs->sectors_per_disk_block_bits); + if (next) + *next = 0; + if (block >= fs->size) + return 0; + ap = addr_2_apage(fs, block, "get_blocklen_at"); + if (unlikely(ap < 0)) + return 0; + map = map_apage(fs, ap, MAP_READ, unlikely(next != NULL) ? &other : NULL); + if (unlikely(IS_ERR(map))) + return 0; + head = get_head(map); + if (unlikely(other != NULL) && !CC_CURRENT(fs, &head->cc, &head->txc)) { + unmap_apage(fs, other, 0); + other = NULL; + } + if (likely(!(head->s.u.l.flags & APAGE_BITMAP))) { + struct aentry *e; + sector_t rs; + int blk = find_block_before(fs, map, block); + if (unlikely(blk < 0)) + goto unmap_0; + e = get_aentry_valid(fs, map, blk); + if (!blk || block >= SPAD2CPU64_LV(&e->start) + SPAD2CPU32_LV(&e->len)) { + if (unlikely(next != NULL)) { + int nextblk = SPAD2CPU16_LV(&e->next); + if (likely(nextblk != 0)) { + e = get_aentry(fs, map, nextblk); + if (unlikely(IS_ERR(e))) + goto unmap_0; + *next = SPAD2CPU64_LV(&e->start); + } else { + *next = SPAD2CPU64_LV(&fs->apage_index[ap].end_sector); + } + } + goto unmap_0; + } + rs = SPAD2CPU64_LV(&e->start) + SPAD2CPU32_LV(&e->len) - block; + if (likely(rs > max)) + ret = max; + else + ret = rs; + } else { + unsigned Xboff = BITMAP_OFFSET(head, block); + unsigned add = 1U << fs->sectors_per_disk_block_bits; + ret = 0; + while (ret < max && + likely(Xboff < BITMAP_SIZE(fs->apage_size)) && + !bmp_test(fs, map, Xboff)) + ret += add, Xboff++; + if (unlikely(next != NULL) && !ret) { + while (likely(Xboff < BITMAP_SIZE(fs->apage_size)) && + bmp_test(fs, map, Xboff)) + block += add, Xboff++; + *next = block; + goto unmap_0; + } + } + if (unlikely(next != NULL)) { + sector_t new_free; + if (unlikely(check_conflict(fs, other, block, ret, &new_free))) { + unsigned orig_size = ret; + unsigned test_len; + if (check_conflict(fs, other, block, 1U << fs->sectors_per_disk_block_bits, &new_free)) { + if (new_free) + *next = new_free; + else + *next = SPAD2CPU64_LV(&fs->apage_index[ap].end_sector); + goto unmap_0; + } + ret = 1U << fs->sectors_per_disk_block_bits; + for (test_len = 2U << fs->sectors_per_disk_block_bits; test_len && test_len < orig_size; test_len <<= 1) { + if (check_conflict(fs, other, block, test_len, &new_free)) + break; + ret = test_len; + } + for (test_len >>= 1; test_len >= 1U << fs->sectors_per_disk_block_bits; test_len >>= 1) { + if (ret + test_len < orig_size && !check_conflict(fs, other, block, ret + test_len, &new_free)) + ret += test_len; + } + goto unmap_ret; + } + } + goto unmap_ret; + +unmap_0: + ret = 0; +unmap_ret: + unmap_apage(fs, map, 0); + if (other) + unmap_apage(fs, other, 0); + return ret; +} + +sector_t spadfs_get_freespace(SPADFS *fs) +{ + return fs->freespace + fs->small_prealloc.n_sectors +#ifdef SPADFS_META_PREALLOC + + fs->meta_prealloc.n_sectors +#endif + ; +} + +static int trim_alloc(SPADFS *fs, struct alloc *al) +{ + sector_t available; + if (unlikely(al->n_sectors + al->extra_sectors < al->n_sectors)) { + al->extra_sectors = -(1U << fs->sectors_per_disk_block_bits) - al->n_sectors; + } + available = spadfs_get_freespace(fs); + if (likely(!capable(CAP_SYS_RESOURCE))) { + if (unlikely(available <= fs->reserve_sectors)) + return 0; + available -= fs->reserve_sectors; + } + if (likely(!(al->flags & ALLOC_METADATA))) { + if (likely(available > fs->group_mask + 1)) + available = fs->group_mask + 1; + if (unlikely(available > fs->max_allocation)) + available = fs->max_allocation; + } + if (unlikely(available < al->n_sectors + al->extra_sectors)) { + if (likely(available >= al->n_sectors)) { + al->extra_sectors = available - al->n_sectors; + } else { + if (unlikely(!available)) + return 0; + al->n_sectors = available; + al->extra_sectors = 0; + if (unlikely(al->flags & ALLOC_METADATA)) + return 0; + } + } + return 1; +} + +static void goal_out_of_fs(SPADFS *fs, struct alloc *al) +{ + unsigned char sectors_per_group_bits = fs->sectors_per_group_bits; + al->sector = (sector_t)fs->zones[2].grp_start << sectors_per_group_bits; + if (unlikely(al->sector >= fs->size)) { + al->sector = (sector_t)fs->zones[1].grp_start << sectors_per_group_bits; + if (unlikely(al->sector >= fs->size)) + al->sector = 0; + } +} + +/* + * Average number of free blocks in group in zone. We couldn't divide 64-bit + * values in kernel, so get approximate value. + */ + +static sector_t zone_average_free(struct spadfszone *z) +{ + sector_t s = z->freespace; + sector_div(s, z->grp_n); + return s; +} + +/* A "slow path" helper for spadfs_alloc_blocks_unlocked */ + +static noinline int alloc_blocks_in_different_group(SPADFS *fs, + struct alloc *al) +{ + sector_t orig_start = al->sector; + struct spadfszone *z; + int orig_zone_empty; + unsigned n_sectors; + int r; + +retry_less: + if (likely(al->flags & ALLOC_BIG_FILE)) + z = &fs->zones[2]; + else if (likely(al->flags & ALLOC_SMALL_FILE)) + z = &fs->zones[1]; + else + z = &fs->zones[0]; + + orig_zone_empty = !z->grp_n; +forward_quad_search: + if (likely(z->grp_n)) { + unsigned pass; + sector_t avg_free = zone_average_free(z); + if (unlikely(avg_free < al->n_sectors)) + avg_free = al->n_sectors; +retry: + for (pass = 0; pass < 2; pass++) { + unsigned i; + unsigned grp_current = orig_start >> + fs->sectors_per_group_bits; + if (unlikely(grp_current < z->grp_start) || + unlikely(grp_current >= z->grp_start + z->grp_n)) + grp_current = z->grp_start + grp_current % z->grp_n; + i = 1; + do { + grp_current += likely(!pass) ? i : 1; + if (unlikely(grp_current >= + z->grp_start + z->grp_n)) + grp_current -= z->grp_n; + if (fs->group_info[grp_current].freespace >= + avg_free) { + al->sector = (sector_t)grp_current << + fs->sectors_per_group_bits; + al->top = (sector_t)(grp_current + 1) << + fs->sectors_per_group_bits; + if (unlikely(al->top < al->sector)) + al->top = fs->size; + r = alloc_blocks_in_range(fs, al); + if (likely(!r)) { +alloc_success_set_group: + al->flags |= + ALLOC_NEW_GROUP_HINT; + return 0; + } + if (unlikely(r != -ENOSPC)) + return r; + } + if (likely(!pass)) + i *= 2; + else + i++; + } while (i <= z->grp_n); + } + if (avg_free > al->n_sectors) { + avg_free = al->n_sectors; + goto retry; + } + } +try_another_zone: + if (likely(al->flags & ALLOC_BIG_FILE)) { + if (likely(z == &fs->zones[2])) + z = &fs->zones[1]; + else if (likely(z == &fs->zones[1])) + z = &fs->zones[0]; + else + goto failed; + } else if (likely(al->flags & ALLOC_SMALL_FILE)) { + if (likely(z == &fs->zones[1])) + z = &fs->zones[2]; + else if (likely(z == &fs->zones[2])) + z = &fs->zones[0]; + else + goto failed; + } else { + if (likely(z == &fs->zones[0])) + z = &fs->zones[1]; + else if (likely(z == &fs->zones[1])) + z = &fs->zones[2]; + else + goto failed; + } + if (z == &fs->zones[2] || orig_zone_empty) + goto forward_quad_search; + if (likely(z->grp_n)) { + unsigned grp_current; + sector_t avg_free = zone_average_free(z) / + SPADFS_AVG_FREE_DIVIDE_OTHERZONE; + if (unlikely(avg_free < al->n_sectors)) + avg_free = al->n_sectors; +retry2: + grp_current = z->grp_start + z->grp_n; + do { + grp_current--; + if (fs->group_info[grp_current].freespace >= avg_free) { + al->sector = (sector_t)grp_current << + fs->sectors_per_group_bits; + al->top = (sector_t)(grp_current + 1) << + fs->sectors_per_group_bits; + if (unlikely(al->top < al->sector)) + al->top = fs->size; + r = alloc_blocks_in_range(fs, al); + if (likely(!r)) + goto alloc_success_set_group; + if (unlikely(r != -ENOSPC)) + return r; + } + } while (grp_current > z->grp_start); + if (avg_free > al->n_sectors) { + avg_free = al->n_sectors; + goto retry2; + } + } + goto try_another_zone; + +failed: + /* + * Try to allocate less sectors + */ + n_sectors = ((al->n_sectors + al->extra_sectors) >> 1) & ~((1U << fs->sectors_per_disk_block_bits) - 1); + if (n_sectors >= al->n_sectors) { + al->extra_sectors = n_sectors - al->n_sectors; + } else { + if (spadfs_discard_all_reservations(fs)) + n_sectors = al->n_sectors; + al->n_sectors = n_sectors; + al->extra_sectors = 0; + } + + /*printk("set max_allocation %x -> %x\n", al->n_sectors, fs->max_allocation);*/ + if (likely(n_sectors < fs->max_allocation)) + fs->max_allocation = n_sectors; + + if (unlikely(!n_sectors) || unlikely(al->flags & ALLOC_METADATA)) + return -ENOSPC; + + if (unlikely(al->n_sectors <= ALLOC_MASK(al->flags))) + al->flags = (al->flags & + ~(ALLOC_MASK_MASK | ALLOC_BIG_FILE)) | + ALLOC_SMALL_FILE; + else if (!al->extra_sectors) + al->n_sectors &= ~ALLOC_MASK(al->flags); + + al->sector = orig_start; + al->top = (al->sector + fs->group_mask + 1) & ~fs->group_mask; + if (unlikely(al->top < al->sector)) + al->top = fs->size; + r = alloc_blocks_in_range(fs, al); + if (!r) + return 0; + if (r != -ENOSPC) + return r; + goto retry_less; +} + +static int spadfs_free_blocks_(SPADFS *fs, sector_t start, sector_t n_sectors, + int acct); + +static int spadfs_alloc_blocks_unlocked(SPADFS *fs, struct alloc *al) +{ + int r; + struct alloc_reservation *res; + + if (unlikely(al->n_sectors & ((1U << fs->sectors_per_disk_block_bits) - 1))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "trying to allocate %u blocks", al->n_sectors); + r = -EFSERROR; + goto ret_r; + } + + if ((res = al->reservation) && res->len) { + unsigned orig_res_len = res->len; + res->len = 0; + al->sector = res->start; + if (al->n_sectors >= orig_res_len) + al->n_sectors = orig_res_len; + al->extra_sectors = 0; + do { + al->top = al->sector + al->n_sectors; + r = alloc_blocks_in_range(fs, al); + if (likely(!r)) { + if (al->n_sectors == orig_res_len) { + spadfs_discard_reservation_unlocked(fs, res); + } else { + res->len = orig_res_len - al->n_sectors; + res->start += al->n_sectors; + } + goto ok_allocated_nores; + } + if (unlikely(r != -ENOSPC)) + goto ret_r; + al->n_sectors >>= 1; + al->n_sectors &= ~((1U << fs->sectors_per_disk_block_bits) - 1); + } while (al->n_sectors); + spadfs_error(fs, TXFLAGS_FS_ERROR, "failed to allocate reserved sectors at (%Lx,%x), flags %x", + (unsigned long long)res->start, + orig_res_len, + al->flags); + spadfs_discard_reservation_unlocked(fs, res); + r = -EFSERROR; + goto ret_r; + } + + al->sector &= ~(sector_t)((1U << fs->sectors_per_disk_block_bits) - 1); + if (unlikely(al->sector >= fs->size)) + goal_out_of_fs(fs, al); + + if (unlikely(al->flags & ALLOC_METADATA)) { + if (al->n_sectors <= (1U << fs->sectors_per_page_bits)) + al->flags |= ((1U << fs->sectors_per_page_bits) - 1) * + ALLOC_MASK_1; + } + + if (al->flags & (ALLOC_PARTIAL_AT_GOAL +#ifdef SPADFS_RESURRECT + | ALLOC_RESURRECT +#endif + )) { + unsigned bl = get_blocklen_at(fs, al->sector, al->n_sectors + al->extra_sectors, NULL); + if (likely(bl != 0)) { + unsigned orig_n_sectors = al->n_sectors; + unsigned orig_extra_sectors = al->extra_sectors; + while (1) { + bl &= ~((1U << fs->sectors_per_disk_block_bits) - 1); + if (bl >= al->n_sectors) { + al->extra_sectors = bl - al->n_sectors; + } else { + if (unlikely(!bl)) + break; + al->n_sectors = bl; + al->extra_sectors = 0; + } + al->top = al->sector + al->n_sectors + al->extra_sectors; + r = alloc_blocks_in_range(fs, al); + if (likely(!r)) + goto ok_allocated; + if (unlikely(r != -ENOSPC)) + goto ret_r; + bl >>= 1; + } + al->n_sectors = orig_n_sectors; + al->extra_sectors = orig_extra_sectors; + } +#ifdef SPADFS_RESURRECT + if (unlikely(al->flags & ALLOC_RESURRECT)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "failed to resurrect blocks at (%Lx,%x)", + (unsigned long long)al->sector, + al->n_sectors); + r = -EFSERROR; + goto ret_r; + } +#endif + } + + if (al->flags & ALLOC_BIG_FILE) { + if (al->n_sectors >= 1U << fs->sectors_per_cluster_bits) { + al->flags |= ((1U << fs->sectors_per_cluster_bits) - 1) * + ALLOC_MASK_1; + al->sector &= ~(sector_t)((1U << fs->sectors_per_cluster_bits) - 1); + } + } + + al->top = (al->sector + fs->group_mask + 1) & ~fs->group_mask; + if (unlikely(al->top < al->sector)) + al->top = fs->size; + + r = alloc_blocks_in_range(fs, al); + if (likely(!r)) + goto ok_allocated; + + if (unlikely(r != -ENOSPC)) + goto ret_r; + + r = alloc_blocks_in_different_group(fs, al); + if (likely(!r)) + goto ok_allocated; +ret_r: + return r; + +ok_allocated: + if ((res = al->reservation)) { + BUG_ON(res->len); + if (likely(al->extra_sectors != 0)) { + res->start = al->sector + al->n_sectors; + res->len = al->extra_sectors; + spadfs_add_reservation(fs, res); + } + } +ok_allocated_nores: + if (unlikely(al->flags & ALLOC_FREE_FROM)) { + r = spadfs_free_blocks_(fs, al->top, + al->sector - al->top, 0); + if (unlikely(r)) + goto ret_r; + } + if (unlikely(al->n_sectors > fs->freespace)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "free block count underrun: %x < %x", + (unsigned)fs->freespace, al->n_sectors); + r = -EFSERROR; + goto ret_r; + } + freespace_decrease(fs, al->sector, al->n_sectors); + return 0; +} + +static pid_t prealloc_pgrp(void) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + return process_group(current); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,30) + struct pid *grp; + pid_t val; + rcu_read_lock(); + grp = task_pgrp(current); + val = pid_vnr(grp); + rcu_read_unlock(); + return val; +#else + return task_pgrp_vnr(current); +#endif +} + +/* + * This is the main entry to block allocator. May be called by any other + * subsystem. + * In order to minimize CPU usage by passing all allocation parameters between + * allocator functions, we pass pointer to struct alloc instead. struct alloc is + * usually placed on caller's stack. + * The following fields in "struct alloc" must be filled before calling this + * function: + * sector: a hint where the allocation should start. Must be dividable by + * "sectors_per_block" value + * n_sectors: preferred number of sectors. Must be dividable by + * "sectors_per_block" value + * flags: + * ALLOC_METADATA: we are allocating metadata, this means: + * - prefer metadata zone + * - alloc exactly n_sectors or nothing (the caller must + * handle failure of allocating more than 1 block without + * loss of functionality --- i.e. create chains instead of + * hash tables in directories) + * - n_sectors must be less or equal than + * "sectors_per_page" + * - the resulting allocation won't cross + * "sectors_per_page" boundary + * ALLOC_SMALL_FILE: we are allocating small file (below threshold) + * - prefer small file zone + * - may alloc less blocks if the disk is too fragmented, + * the resulting number is returned in n_sectors. The + * result is always multiple of "sectors_per_block" + * ALLOC_BIG_FILE: we are allocating big file + * - prefer big file zone + * - prefer allocating at sectors_per_cluster boundary + * - may alloc less blocks if the disk is too fragmented, + * the resulting number is returned in n_sectors. The + * result is always multiple of "sectors_per_block" + * ALLOC_PARTIAL_AT_GOAL: may be ORed with other flags. It means + * that if the block at al->sector is free, we always + * allocate from al->sector, even if we allocate less than + * n_sectors sectors --- this is used when extending file + * with non-zero length + * Returns: + * -ENOSPC --- the blocks couldn't be allocated + * If ALLOC_SMALL_FILE or ALLOC_BIG_FILE was specified, it means + * that the disk is totally full + * If ALLOC_METADATA was specified and n_sectors == + * "sectors_per_page", it also means that the disk is totally full + * If ALLOC_METADATA was specified and n_sectors is larger, it + * means that the disk is too fragmented to allocate specified + * number of blocks + * -EIO, -EFSERROR, -E... --- some error occured + * 0 --- succesful, then + * al->sector --- allocated sector + * al->n_sectors --- allocated number of sectors + * al->flags & ALLOC_NEW_GROUP_HINT --- the caller should set group + * hint for the file's directory to returned value + */ + +int spadfs_alloc_blocks(SPADFS *fs, struct alloc *al) +{ + int r; + struct prealloc_state *prealloc; + unsigned max_prealloc; + + mutex_lock(&fs->alloc_lock); + + if (unlikely(!trim_alloc(fs, al))) { + r = -ENOSPC; + goto unlock_ret_r; + } + + max_prealloc = (fs->max_prealloc & ~((512U << fs->sectors_per_disk_block_bits) - 1)) >> 9; + if (unlikely(max_prealloc > fs->group_mask + 1)) + max_prealloc = fs->group_mask + 1; + if (unlikely(max_prealloc > fs->max_allocation)) + max_prealloc = fs->max_allocation; + + if (al->reservation && al->reservation->len) + goto no_prealloc; + + if ((al->flags & (ALLOC_SMALL_FILE | ALLOC_PARTIAL_AT_GOAL +#ifdef SPADFS_RESURRECT + | ALLOC_RESURRECT +#endif + )) == (ALLOC_SMALL_FILE | ALLOC_PARTIAL_AT_GOAL)) { + if (al->sector == fs->small_prealloc.sector && fs->small_prealloc.n_sectors && + prealloc_pgrp() == fs->small_prealloc.pgrp) { + if (al->n_sectors > fs->small_prealloc.n_sectors) + al->n_sectors = fs->small_prealloc.n_sectors; + fs->small_prealloc.sector += al->n_sectors; + fs->small_prealloc.n_sectors -= al->n_sectors; + r = 0; + goto unlock_ret_r; + } + } + + if (al->flags & (ALLOC_BIG_FILE | ALLOC_PARTIAL_AT_GOAL +#ifdef SPADFS_RESURRECT + | ALLOC_RESURRECT +#endif + ) || unlikely(!max_prealloc)) + goto no_prealloc; +#ifdef SPADFS_META_PREALLOC + else if (al->flags & ALLOC_METADATA) + prealloc = &fs->meta_prealloc; +#endif + else if (al->flags & ALLOC_SMALL_FILE) + prealloc = &fs->small_prealloc; + else + goto no_prealloc; + + { + struct alloc pre_al; + pid_t pgrp = prealloc_pgrp(); + int retried = 0; +retry_prealloc: + if (unlikely(!prealloc->n_sectors)) { + unsigned long j = jiffies; + if (j - prealloc->last_alloc > PREALLOC_TIMEOUT || pgrp != prealloc->pgrp) { + prealloc->last_alloc = j; + prealloc->n_allocations = 0; + prealloc->allocations_size = 0; + } + prealloc->n_allocations++; + prealloc->allocations_size += al->n_sectors; + prealloc->pgrp = pgrp; + /*printk("time: %lu, flags %x, n_allocations %u, allocations_size %u\n", j - prealloc->last_alloc, al->flags, prealloc->n_allocations, prealloc->allocations_size);*/ + if (prealloc->n_allocations >= PREALLOC_THRESHOLD) { + pre_al.sector = al->sector; + pre_al.flags = al->flags; + pre_al.n_sectors = prealloc->allocations_size; + if (pre_al.n_sectors > max_prealloc) + pre_al.n_sectors = max_prealloc; + if (unlikely(pre_al.n_sectors > fs->max_allocation)) + goto skip_prealloc; + pre_al.extra_sectors = 0; + pre_al.reservation = NULL; + if (likely(pre_al.n_sectors >= al->n_sectors)) { + r = spadfs_alloc_blocks_unlocked(fs, &pre_al); + if (!r) { + /*printk("prealloc done: %llx, %x\n", (unsigned long long)pre_al.sector, pre_al.n_sectors);*/ + prealloc->sector = pre_al.sector; + prealloc->n_sectors = pre_al.n_sectors; + } + } + skip_prealloc:; + } + } else { + if (unlikely(jiffies - prealloc->last_alloc > PREALLOC_DISCARD_TIMEOUT)) + spadfs_prealloc_discard_unlocked(fs, prealloc); + } + if (unlikely(prealloc->n_sectors) && pgrp == prealloc->pgrp) { +retry_use_prealloc: + if (likely(al->n_sectors <= prealloc->n_sectors)) { +#ifdef SPADFS_META_PREALLOC + if (al->flags & ALLOC_METADATA) { + unsigned sectors_per_page = 1U << fs->sectors_per_page_bits; + unsigned to_page = sectors_per_page - ((unsigned)prealloc->sector & (sectors_per_page - 1)); + if (unlikely(to_page < al->n_sectors)) { + if (unlikely(to_page > prealloc->n_sectors)) + to_page = prealloc->n_sectors; + spadfs_free_blocks_unlocked(fs, prealloc->sector, to_page); + prealloc->sector += to_page; + prealloc->n_sectors -= to_page; + goto retry_use_prealloc; + } + } +#endif + al->sector = prealloc->sector; + prealloc->sector += al->n_sectors; + prealloc->n_sectors -= al->n_sectors; + r = 0; + prealloc->last_alloc = jiffies; + prealloc->allocations_size += al->n_sectors; + /*printk("taking from prealloc: flags %x, n_allocations %u, allocations_size %u took %u left %u\n", al->flags, prealloc->n_allocations, prealloc->allocations_size, al->n_sectors, prealloc->n_sectors);*/ + goto unlock_ret_r; + } else { + unsigned need, bl; + need = al->n_sectors - prealloc->n_sectors; + bl = get_blocklen_at(fs, prealloc->sector + prealloc->n_sectors, need, NULL); + if (bl >= need) { + pre_al.sector = prealloc->sector + prealloc->n_sectors; + pre_al.n_sectors = need; + if (pre_al.n_sectors < prealloc->allocations_size && pre_al.n_sectors < max_prealloc) { + pre_al.n_sectors = prealloc->allocations_size; + if (pre_al.n_sectors > max_prealloc) + pre_al.n_sectors = max_prealloc; + } + pre_al.extra_sectors = 0; + pre_al.flags = al->flags | ALLOC_PARTIAL_AT_GOAL; + pre_al.reservation = NULL; + r = spadfs_alloc_blocks_unlocked(fs, &pre_al); + if (!r) { + /*printk("prealloc extend: %llx, %x\n", (unsigned long long)pre_al.sector, pre_al.n_sectors);*/ + if (likely(pre_al.sector == prealloc->sector + prealloc->n_sectors)) { + prealloc->n_sectors += pre_al.n_sectors; + } else { + spadfs_free_blocks_unlocked(fs, prealloc->sector, prealloc->n_sectors); + prealloc->sector = pre_al.sector; + prealloc->n_sectors = pre_al.n_sectors; + } + if (likely(al->n_sectors <= prealloc->n_sectors)) + goto retry_use_prealloc; + } + } + spadfs_free_blocks_unlocked(fs, prealloc->sector, prealloc->n_sectors); + prealloc->sector = 0; + prealloc->n_sectors = 0; + if (!retried) { + retried = 1; + goto retry_prealloc; + } + } + } + } + +no_prealloc: + + r = spadfs_alloc_blocks_unlocked(fs, al); + if (r == -ENOSPC) { + if ( +#ifdef SPADFS_META_PREALLOC + fs->meta_prealloc.n_sectors || +#endif + fs->small_prealloc.n_sectors) { +#ifdef SPADFS_META_PREALLOC + spadfs_prealloc_discard_unlocked(fs, &fs->meta_prealloc); +#endif + spadfs_prealloc_discard_unlocked(fs, &fs->small_prealloc); + + r = spadfs_alloc_blocks_unlocked(fs, al); + } + } + +unlock_ret_r: + + if (likely(!r) && unlikely(fs->trim_len != 0)) { + while (al->sector < fs->trim_start + fs->trim_len && + al->sector + al->n_sectors > fs->trim_start) { + mutex_unlock(&fs->alloc_lock); + msleep(1); + mutex_lock(&fs->alloc_lock); + } + } + + mutex_unlock(&fs->alloc_lock); + + return r; +} + +void spadfs_prealloc_discard_unlocked(SPADFS *fs, struct prealloc_state *prealloc) +{ + if (prealloc->n_sectors) + spadfs_free_blocks_unlocked(fs, prealloc->sector, prealloc->n_sectors); + prealloc->sector = 0; + prealloc->n_sectors = 0; + prealloc->last_alloc = jiffies; + prealloc->n_allocations = 0; + prealloc->allocations_size = 0; +} + +static void adjust_max_run(SPADFS *fs, unsigned n_sectors) +{ + if (unlikely(n_sectors > fs->max_freed_run)) + fs->max_freed_run = n_sectors; +} + +/* Free a given sector run in mapped apage */ + +static int apage_free(SPADFS *fs, APAGE_MAP *map, sector_t off, u32 len) +{ + struct apage_head *head = get_head(map); + if (likely(!(head->s.u.l.flags & APAGE_BITMAP))) { + int preblk, postblk, newblk; + struct aentry *pre, *post, *new; + preblk = find_block_before(fs, map, off); + if (unlikely(preblk < 0)) + return preblk; + pre = get_aentry_valid(fs, map, preblk); + postblk = SPAD2CPU16_LV(&pre->next); /* already valid */ + post = get_aentry_valid(fs, map, postblk); + if (likely(preblk != 0)) { + if ((sector_t)SPAD2CPU64_LV(&pre->start) + + SPAD2CPU32_LV(&pre->len) == off) { + u32 new_pre_len = SPAD2CPU32_LV(&pre->len) + len; + if (unlikely(new_pre_len < (u32)len)) + goto nj1; + if (likely(postblk != 0)) { + if (off + len == + (sector_t)SPAD2CPU64_LV(&post->start) + ) { + u32 new_post_len = new_pre_len + SPAD2CPU32_LV(&post->len); + if (unlikely(new_post_len < new_pre_len)) + goto nj2; + CPU2SPAD32_LV(&pre->len, new_post_len); + adjust_max_run(fs, new_post_len); + delete_block(fs, map, post); + goto done; + } else if (unlikely(off + len > + (sector_t)SPAD2CPU64_LV(&post->start))) { + goto post_over; + } + } +nj2: + CPU2SPAD32_LV(&pre->len, new_pre_len); + adjust_max_run(fs, new_pre_len); + goto done; + } else if (unlikely((sector_t)SPAD2CPU64_LV(&pre->start) + + SPAD2CPU32_LV(&pre->len) > off)) { + postblk = preblk; + post = pre; + goto post_over; + } + } +nj1: + if (likely(postblk != 0)) { + if (off + len == + (sector_t)SPAD2CPU64_LV(&post->start)) { + u32 new_post_len = SPAD2CPU32_LV(&post->len) + len; + if (unlikely(new_post_len < (u32)len)) + goto nj3; + CPU2SPAD64_LV(&post->start, off); + CPU2SPAD32_LV(&post->len, new_post_len); + adjust_max_run(fs, new_post_len); + goto done; + } else if (unlikely(off + len > + (sector_t)SPAD2CPU64_LV(&post->start))) { +post_over: + spadfs_error(fs, TXFLAGS_FS_ERROR, + "free (%Lx,%x) does overlap with block (%Lx,%x)", + (unsigned long long)off, + (unsigned)len, + (unsigned long long)SPAD2CPU64_LV( + &post->start), + (unsigned)SPAD2CPU32_LV(&post->len)); + return -EFSERROR; + } + } +nj3: + if (unlikely(!(newblk = SPAD2CPU16_LV(&head->s.u.l.freelist)))) + return 1; + new = get_aentry(fs, map, newblk); + if (unlikely(IS_ERR(new))) + return PTR_ERR(new); + head->s.u.l.freelist = new->next; + CPU2SPAD64_LV(&new->start, off); + CPU2SPAD32_LV(&new->len, len); + CPU2SPAD16_LV(&new->prev, preblk); + CPU2SPAD16_LV(&new->next, postblk); + CPU2SPAD16_LV(&pre->next, newblk); + CPU2SPAD16_LV(&post->prev, newblk); + adjust_max_run(fs, len); + +done: + return 0; + } else { + unsigned bmpoff; + adjust_max_run(fs, len); + bmpoff = BITMAP_OFFSET(head, off); + len = BITMAP_LEN(head, len); + if (unlikely(bmpoff + len <= bmpoff) || + unlikely(bmpoff + len > BITMAP_SIZE(fs->apage_size))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "bad bitmap offset: %Lx,%x free(%Lx,%x)", + (unsigned long long)MAKE_D_OFF( + head->s.u.b.start0, head->s.u.b.start1), + bmpoff, + (unsigned long long)off, (unsigned)len); + return -EFSERROR; + } + do { + bmp_clear(fs, map, bmpoff); + bmpoff++; + } while (--len); + return 0; + } +} + +__cold static int split_apage(SPADFS *fs, APAGE_MAP *map0, APAGE_MAP *map0_other, int ap); +__cold static int convert_to_bitmap(SPADFS *fs, APAGE_MAP *map, sector_t start, sector_t end); + +/* + * General block-freeing routine. + * spadfs_free_blocks should be called instead of this + * --- it locks the semaphore and forces "acct" to 1. + * + * start and n_sectors is the extent. acct means that we should do free space + * accounting. The only situation where acct could be 0 is the call from + * spadfs_alloc_blocks (in rare case when spadfs_alloc_blocks needs to allocate + * blocks in the middle of one free extent, the extent must be split to two --- + * instead of complicating logic of spadfs_alloc_blocks, it allocates more space + * from the beginning of the extent and then calls spadfs_free_blocks_ with + * acct == 0) + */ + +static int spadfs_free_blocks_(SPADFS *fs, sector_t start, sector_t n_sectors, int acct) +{ + int ap; + int r; + APAGE_MAP *map, *other; + if (unlikely(!validate_range(fs->size, (1U << fs->sectors_per_disk_block_bits) - 1, start, n_sectors))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "freeing blocks: %Lx, %Lx", + (unsigned long long)start, + (unsigned long long)n_sectors); + return -EFSERROR; + } +retry: + if (unlikely((n_sectors & ~(sector_t)0xffffffffu) != 0)) { + /* + * Max block size is 64k, that is 128 sectors. + * Keep n_sectors aligned. + */ + const u32 max_sectors = -(1U << MAX_SECTORS_PER_BLOCK_BITS); + if (unlikely(r = spadfs_free_blocks_(fs, start, max_sectors, + acct))) + return r; + start += max_sectors; + n_sectors -= max_sectors; + goto retry; + } +retry_32: + ap = addr_2_apage(fs, start, "spadfs_free_blocks_"); + if (unlikely(ap < 0)) + return ap; + if ((sector_t)SPAD2CPU64_LV(&fs->apage_index[ap].end_sector) < + start + n_sectors) { + u32 ss = (sector_t)SPAD2CPU64_LV( + &fs->apage_index[ap].end_sector) - start; + if (unlikely(!ss) || unlikely(ss >= n_sectors)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "corrupted apage index: freeing %Lx,%Lx - apage %d, apage_end %Lx", + (unsigned long long)start, + (unsigned long long)n_sectors, + ap, + (unsigned long long)SPAD2CPU64_LV( + &fs->apage_index[ap].end_sector)); + return -EFSERROR; + } + if (unlikely(r = spadfs_free_blocks_(fs, start, ss, acct))) + return r; + start += ss; + n_sectors -= ss; + goto retry_32; + } + map = map_apage(fs, ap, MAP_WRITE, &other); + if (unlikely(IS_ERR(map))) + return PTR_ERR(map); +free_again: + r = apage_free(fs, map, start, n_sectors); + if (likely(r <= 0)) { + unmap_apage(fs, map, 1); + unmap_apage(fs, other, 0); + if (likely(!r) && likely(acct)) { + freespace_increase(fs, start, n_sectors); + } + return r; + } + if (likely((sector_t)SPAD2CPU64_LV(&fs->apage_index[ap].end_sector) - + (sector_t)SPAD2CPU64_LV(&fs->apage_index[ap - 1].end_sector) > + BITMAP_SIZE(fs->apage_size) << fs->sectors_per_disk_block_bits)) { + if (unlikely(r = split_apage(fs, map, other, ap))) + return r; + goto retry_32; + } + if (unlikely(r = convert_to_bitmap(fs, map, + (sector_t)SPAD2CPU64_LV(&fs->apage_index[ap - 1].end_sector), + (sector_t)SPAD2CPU64_LV(&fs->apage_index[ap].end_sector)))) { + unmap_apage(fs, map, 1); + unmap_apage(fs, other, 0); + return r; + } + goto free_again; +} + +int spadfs_free_blocks_unlocked(SPADFS *fs, sector_t start, + sector_t n_sectors) +{ + return spadfs_free_blocks_(fs, start, n_sectors, 1); +} + +static int spadfs_free_blocks(SPADFS *fs, sector_t start, sector_t n_sectors) +{ + int r; + mutex_lock(&fs->alloc_lock); + r = spadfs_free_blocks_unlocked(fs, start, n_sectors); + mutex_unlock(&fs->alloc_lock); + return r; +} + +int spadfs_free_blocks_metadata(SPADFS *fs, sector_t start, + sector_t n_sectors) +{ + spadfs_discard_buffers(fs, start, n_sectors); + return spadfs_free_blocks(fs, start, n_sectors); +} + +static int write_apage_index(SPADFS *fs); + +/* Split apage to two */ + +__cold static noinline int split_apage(SPADFS *fs, APAGE_MAP *map0, APAGE_MAP *map0_other, int ap) +{ + int r; + unsigned i, n; + struct aentry *ae; + sector_t split_block; + APAGE_MAP *map1, *map1_other; + struct apage_index_entry aie; + unsigned b_s; + if (unlikely(fs->n_active_apages >= fs->n_apages)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "all %u apages used", fs->n_active_apages); + r = -EFSERROR; + goto ret0; + } + n = 0; + ae = (struct aentry *)get_head(map0); + for (i = fs->apage_size / (2 * sizeof(struct aentry)); i; i--) { + n = SPAD2CPU16_LV(&ae->next); + ae = get_aentry(fs, map0, n); + if (unlikely(IS_ERR(ae))) { + r = PTR_ERR(ae); + goto ret0; + } + } + if (unlikely(!n)) + split_block = ((sector_t)SPAD2CPU64_LV(&fs->apage_index[ap - 1].end_sector) + + (sector_t)SPAD2CPU64_LV(&fs->apage_index[ap].end_sector)) + >> 1; + else + split_block = SPAD2CPU64_LV(&ae->start); + split_block &= ~(sector_t)((1U << fs->sectors_per_disk_block_bits) - 1); + + /*printk("spadfs: splitting apage %d : (%llx %llx %llx)\n", ap, (unsigned long long)SPAD2CPU64_LV(&fs->apage_index[ap - 1].end_sector), (unsigned long long)split_block, (unsigned long long)SPAD2CPU64_LV(&fs->apage_index[ap].end_sector));*/ + + b_s = (BITMAP_SIZE(fs->apage_size) >> 1 << fs->sectors_per_disk_block_bits); + if (unlikely(split_block - (sector_t)SPAD2CPU64_LV(&fs->apage_index[ap - 1].end_sector) < b_s)) { + split_block = (sector_t)SPAD2CPU64_LV(&fs->apage_index[ap - 1].end_sector) + b_s; + split_block += (1U << fs->sectors_per_disk_block_bits) - 1; + split_block &= ~(sector_t)((1U << fs->sectors_per_disk_block_bits) - 1); + } + if (unlikely((sector_t)SPAD2CPU64_LV(&fs->apage_index[ap].end_sector) - + split_block < b_s)) { + split_block = (sector_t)SPAD2CPU64_LV(&fs->apage_index[ap].end_sector) - b_s; + split_block &= ~(sector_t) ((1U << fs->sectors_per_disk_block_bits) - 1); + } + copy_apage(fs, fs->tmp_map, map0); + map1 = map_apage(fs, fs->n_active_apages, MAP_NEW, &map1_other); + if (unlikely(IS_ERR(map1))) { + r = PTR_ERR(map1); + goto ret0; + } + copy_apage(fs, map1_other, map0_other); + unmap_apage(fs, map1_other, 1); + make_apage(fs, map0); + make_apage(fs, map1); + for (i = sizeof(struct aentry); i < fs->apage_size; + i += sizeof(struct aentry)) { + sector_t st; + struct aentry *ae = get_aentry_valid(fs, fs->tmp_map, i); + if (unlikely(ae->len == CPU2SPAD32_CONST(0))) + continue; + st = SPAD2CPU64_LV(&ae->start); + if (st >= split_block) + r = apage_free(fs, map1, st, SPAD2CPU32_LV(&ae->len)); + else if (st + SPAD2CPU32_LV(&ae->len) <= split_block) + r = apage_free(fs, map0, st, SPAD2CPU32_LV(&ae->len)); + else { + r = apage_free(fs, map0, st, split_block - st); + if (likely(!r)) + r = apage_free(fs, map1, split_block, + st + SPAD2CPU32_LV(&ae->len) - + split_block); + } + if (unlikely(r)) { + if (r > 0) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "out of space when splitting apage"); + r = -EFSERROR; + } + unmap_apage(fs, map1, 1); + goto ret0; + } + } + unmap_apage(fs, map0, 1); + unmap_apage(fs, map1, 1); + unmap_apage(fs, map0_other, 0); + aie = fs->apage_index[fs->n_active_apages]; + for (i = fs->n_active_apages; i > ap + 1; i--) { + fs->apage_index[i] = fs->apage_index[i - 1]; + spadfs_cond_resched(); + } + fs->apage_index[ap + 1] = aie; + fs->apage_index[ap + 1].end_sector = + (sector_t)fs->apage_index[ap].end_sector; + CPU2SPAD64_LV(&fs->apage_index[ap].end_sector, split_block); + fs->n_active_apages++; + + spadfs_prune_cached_apage_buffers(fs); + + return write_apage_index(fs); + +ret0: + unmap_apage(fs, map0, 1); + unmap_apage(fs, map0_other, 0); + return r; +} + +/* Write apage index after the split */ + +__cold static int write_apage_index(SPADFS *fs) +{ + unsigned i, n; + sector_t sec; + if (likely(!CC_CURRENT(fs, &fs->a_cc, &fs->a_txc))) { + struct txblock *tx; + struct buffer_head *bh; + tx = spadfs_read_tx_block(fs, &bh, "write_apage_index 1"); + if (unlikely(IS_ERR(tx))) + return PTR_ERR(tx); + start_atomic_buffer_modify(fs, bh); + CC_SET_CURRENT(fs, &fs->a_cc, &fs->a_txc); + tx->a_cc = fs->a_cc; + tx->a_txc = fs->a_txc; + spadfs_tx_block_checksum(tx); + end_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + } + n = APAGE_INDEX_SECTORS(fs->n_apages, 512U << fs->sectors_per_disk_block_bits); + sec = CC_VALID(fs, &fs->a_cc, &fs->a_txc) ? fs->apage_index0_sec : fs->apage_index1_sec; + for (i = 0; i < n; + i += 1U << fs->sectors_per_buffer_bits, + sec += 1U << fs->sectors_per_buffer_bits) { + struct buffer_head *bh; + void *p; + if (fs->split_happened && + ((unsigned long)i << 9) >= (unsigned long)fs->n_active_apages * sizeof(struct apage_index_entry)) + break; + p = spadfs_get_new_sector(fs, sec, &bh, "write_apage_index 2"); + if (unlikely(IS_ERR(p))) + continue; + memcpy(p, (u8 *)fs->apage_index + ((unsigned long)i << 9), 512U << fs->sectors_per_buffer_bits); + mark_buffer_dirty(bh); + spadfs_brelse(fs, bh); + } + fs->split_happened = 1; + return 0; +} + +/* Convert apage to bitmap */ + +__cold static noinline int convert_to_bitmap(SPADFS *fs, APAGE_MAP *map, sector_t start, sector_t end) +{ + unsigned i; + copy_apage(fs, fs->tmp_map, map); + make_apage_bitmap(fs, map, start); + for (i = sizeof(struct aentry); i < fs->apage_size; + i += sizeof(struct aentry)) { + struct aentry *ae = get_aentry_valid(fs, fs->tmp_map, i); + sector_t st; + unsigned len = SPAD2CPU32_LV(&ae->len); + unsigned Xoff, Xlen; + if (unlikely(!len)) + continue; + st = SPAD2CPU64_LV(&ae->start); + if (unlikely(st < start) || + unlikely(st + len < st) || + unlikely(st + len > end)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "pointer (%Lx,%x) out of range (%Lx-%Lx)", + (unsigned long long)st, + len, + (unsigned long long)start, + (unsigned long long)end); + return -EFSERROR; + } + if (unlikely(((unsigned)st | (unsigned)len) & ((1U << fs->sectors_per_disk_block_bits) - 1))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "pointer (%Lx,%x) not aligned", + (unsigned long long)st, + len); + } + Xoff = (st - start) >> fs->sectors_per_disk_block_bits; + Xlen = len >> fs->sectors_per_disk_block_bits; + do { + if (unlikely(!bmp_test(fs, map, Xoff))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "block %Lx+%x freed twice", + (unsigned long long)st, + (Xoff << fs->sectors_per_disk_block_bits) - (unsigned)(st - start)); + return -EFSERROR; + } + bmp_clear(fs, map, Xoff); + Xoff++; + } while (--Xlen); + } + return 0; +} + +void spadfs_reclaim_max_allocation(SPADFS *fs) +{ + unsigned max_allocation; + mutex_lock(&fs->alloc_lock); + max_allocation = fs->max_allocation * 2 + (4096U << fs->sectors_per_disk_block_bits); + if (likely(max_allocation >= fs->max_allocation)) { + if (max_allocation < fs->freespace) + max_allocation = fs->freespace; + fs->max_allocation = max_allocation; + } + mutex_unlock(&fs->alloc_lock); +} + +#ifdef SPADFS_FSTRIM +int spadfs_trim_fs(SPADFS *fs, u64 start, u64 end, u64 minlen, sector_t *result) +{ + int err = 0; + unsigned max; + sector_t next; + *result = 0; + start &= -(u64)(1U << fs->sectors_per_disk_block_bits); + end &= -(u64)(1U << fs->sectors_per_disk_block_bits); + if (!end || end > fs->size) + end = fs->size; + if (!minlen) + minlen = 1; + + if (READ_ONCE(fs->need_background_sync)) { + spadfs_commit(fs); + } + + while (start < end && !err) { + sync_lock_decl + + mutex_lock(&fs->trim_lock); + down_read_sync_lock(fs); + mutex_lock(&fs->alloc_lock); + + if (unlikely(sb_rdonly(fs->s))) { + err = -EROFS; + mutex_unlock(&fs->alloc_lock); + up_read_sync_lock(fs); + mutex_unlock(&fs->trim_lock); + break; + } + if (end - start == (unsigned)(end - start)) + max = end - start; + else + max = UINT_MAX; + fs->trim_len = get_blocklen_at(fs, start, max, &next); + if (fs->trim_len) { + fs->trim_start = start; + start += fs->trim_len; + } else { + if (next <= start) + start = end; + else + start = next; + } + + mutex_unlock(&fs->alloc_lock); + up_read_sync_lock(fs); + + if (fs->trim_len >= minlen) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,19,0) + err = blkdev_issue_discard(fs->s->s_bdev, fs->trim_start, fs->trim_len, GFP_NOFS, 0); +#else + err = blkdev_issue_discard(fs->s->s_bdev, fs->trim_start, fs->trim_len, GFP_NOFS); +#endif + if (likely(!err)) + *result += fs->trim_len; + } + + mutex_lock(&fs->alloc_lock); + fs->trim_start = 0; + fs->trim_len = 0; + mutex_unlock(&fs->alloc_lock); + mutex_unlock(&fs->trim_lock); + + if (!err && fatal_signal_pending(current)) + err = -EINTR; + } + return err; +} +#endif diff --git a/fs/spadfs/allocmem.c b/fs/spadfs/allocmem.c new file mode 100644 index 000000000..ec09be2fb --- /dev/null +++ b/fs/spadfs/allocmem.c @@ -0,0 +1,217 @@ +#include "spadfs.h" + +void spadfs_allocmem_init(SPADFS *fs) +{ + fs->alloc_mem = RB_ROOT; + fs->alloc_mem_sectors = 0; +} + +void spadfs_allocmem_done(SPADFS *fs) +{ + struct rb_node *root; + while ((root = fs->alloc_mem.rb_node)) { +#define am rb_entry(root, struct allocmem, rb_node) + fs->alloc_mem_sectors -= am->len; + rb_erase(root, &fs->alloc_mem); + kmem_cache_free(spadfs_extent_cachep, am); +#undef am + } + BUG_ON(fs->alloc_mem_sectors != 0); +} + +static struct allocmem *spadfs_allocmem_find_node(SPADFS *fs, sector_t off, + sector_t n_sec, + sector_t *next_allocated) +{ + struct rb_node *p = fs->alloc_mem.rb_node; + while (p) { +#define am rb_entry(p, struct allocmem, rb_node) + if (off + n_sec <= am->start) { + *next_allocated = am->start; + p = am->rb_node.rb_left; + } else if (likely(off >= am->start + am->len)) { + p = am->rb_node.rb_right; + } else { + return am; + } +#undef am + } + return NULL; +} + +int spadfs_allocmem_find(SPADFS *fs, sector_t off, sector_t n_sec, + sector_t *next_change) +{ + struct allocmem *am; + *next_change = 0; + am = spadfs_allocmem_find_node(fs, off, n_sec, next_change); + if (unlikely(am != NULL)) { + *next_change = am->start + am->len; + return 1; + } + return 0; +} + +int spadfs_allocmem_add(SPADFS *fs, sector_t off, sector_t n_sec) +{ + struct rb_node **p = &fs->alloc_mem.rb_node; + struct rb_node *parent = NULL; + struct allocmem *new; + while (*p) { + parent = *p; +#define am rb_entry(parent, struct allocmem, rb_node) + if (off + n_sec < am->start) { + p = &am->rb_node.rb_left; + } else if (off > am->start + am->len) { + p = &am->rb_node.rb_right; + } else if (off + n_sec == am->start) { + am->start -= n_sec; + am->len += n_sec; + goto ret_0; + } else if (likely(off == am->start + am->len)) { + am->len += n_sec; + goto ret_0; + } else { + printk(KERN_EMERG "spadfs: new alloc mem entry %Lx,%Lx overlaps with %Lx,%Lx\n", + (unsigned long long)off, + (unsigned long long)(off + n_sec), + (unsigned long long)am->start, + (unsigned long long)(am->start + am->len)); + BUG(); + } +#undef am + } + new = kmem_cache_alloc(spadfs_extent_cachep, GFP_NOFS); + if (unlikely(!new)) + return -ENOMEM; + new->start = off; + new->len = n_sec; + rb_link_node(&new->rb_node, parent, p); + rb_insert_color(&new->rb_node, &fs->alloc_mem); +ret_0: + fs->alloc_mem_sectors += n_sec; + return 0; +} + +void spadfs_allocmem_delete(SPADFS *fs, sector_t off, sector_t n_sec) +{ + do { + sector_t step_n_sec; + sector_t d_off_sink; + struct allocmem *am = spadfs_allocmem_find_node(fs, off, 1, + &d_off_sink); + if (unlikely(!am)) { + printk(KERN_EMERG "spadfs: trying to free non-present block %Lx,%Lx\n", + (unsigned long long)off, + (unsigned long long)n_sec); + BUG(); + } + if (unlikely(off < am->start)) { + printk(KERN_EMERG "spadfs: trying to free non-present block %Lx, %Lx, found %Lx,%Lx\n", + (unsigned long long)off, + (unsigned long long)n_sec, + (unsigned long long)am->start, + (unsigned long long)am->len); + BUG(); + } + + step_n_sec = n_sec; + if (unlikely(step_n_sec > (am->start + am->len) - off)) + step_n_sec = (am->start + am->len) - off; + + if (off == am->start) { + am->start += step_n_sec; + am->len -= step_n_sec; + fs->alloc_mem_sectors -= step_n_sec; + if (!am->len) { + rb_erase(&am->rb_node, &fs->alloc_mem); + kmem_cache_free(spadfs_extent_cachep, am); + } + } else if (off + step_n_sec == am->start + am->len) { + am->len -= step_n_sec; + fs->alloc_mem_sectors -= step_n_sec; + } else { + sector_t orig_len = am->len; + am->len = off - am->start; + fs->alloc_mem_sectors -= orig_len - am->len; + if (unlikely(spadfs_allocmem_add(fs, off + step_n_sec, + am->start + orig_len - (off + step_n_sec)))) { + fs->alloc_mem_sectors += orig_len - am->len; + am->len = orig_len; + printk(KERN_WARNING "spadfs: memory allocation failure, leaking allocmem\n"); + } + } + off += step_n_sec; + n_sec -= step_n_sec; + } while (unlikely(n_sec != 0)); +} + +#ifdef CHECK_ALLOCMEM + +int spadfs_allocmem_unit_test(SPADFS *fs) +{ + u8 map[256]; + unsigned x; + sector_t zzzz; + printk("allocmem unit test\n"); + memset(map, 0, sizeof map); + for (x = 0; x < 100000; x++) { + unsigned y; + unsigned z; + unsigned zz; + unsigned xp; + get_random_bytes(&y, sizeof y); + y %= sizeof(map); + get_random_bytes(&z, sizeof z); + z = z % (sizeof(map) - y) + 1; + for (zz = 0; zz < z; zz++) { + if (map[y + zz] != map[y]) break; + } + for (z = 0; z < zz; z++) + map[y + z] ^= 1; + if (map[y]) { + if (spadfs_allocmem_add(fs, y, zz)) { + printk("spadfs_allocmem_add failed\n"); + return -ENOMEM; + } + } else { + spadfs_allocmem_delete(fs, y, zz); + } + get_random_bytes(&y, sizeof y); + y %= sizeof(map); + get_random_bytes(&z, sizeof z); + z = z % (sizeof(map) - y) + 1; + get_random_bytes(&xp, sizeof xp); + if (z > 5 && xp & 1) z = (z % 5) + 1; + xp = 0; + for (zz = 0; zz < z; zz++) { + if (map[y + zz]) xp = 1; + } + zzzz = 0; + if (spadfs_allocmem_find(fs, y, z, &zzzz) != xp) { + printk("unit test error for %x/%x != %u\n", y, z, xp); + return -EINVAL; + } + if (xp && zzzz <= y) { + printk("next free returned wrong: %x < %x\n", + (unsigned)zzzz, y); + return -EINVAL; + } + } + for (x = 0; x < sizeof(map); x++) { + if (spadfs_allocmem_find(fs, x, 1, &zzzz) != map[x]) { + printk("final test failed at %x\n", x); + return -EINVAL; + } + if (map[x]) spadfs_allocmem_delete(fs, x, 1); + } + if (fs->alloc_mem_sectors) { + printk("allocmem block count leaked: %Lx\n", + (unsigned long long)fs->alloc_mem_sectors); + return -EINVAL; + } + printk("allocmem unit test passed\n"); + return 0; +} + +#endif diff --git a/fs/spadfs/buffer.c b/fs/spadfs/buffer.c new file mode 100644 index 000000000..c556faf13 --- /dev/null +++ b/fs/spadfs/buffer.c @@ -0,0 +1,350 @@ +#include "spadfs.h" + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,3,0) +#define NEW_BIO +#endif + +#ifdef CHECK_BUFFER_LEAKS +static void spadfs_track_buffer(SPADFS *fs, struct buffer_head *bh, + const char *msg); +#else +#define spadfs_track_buffer(fs, bh, msg) do { } while (0) +#endif + +static void *spadfs_read_sector_physical(SPADFS *fs, sector_t secno, + struct buffer_head **bhp, unsigned ahead, + const char *msg); + +void *spadfs_read_sector(SPADFS *fs, sector_t secno, struct buffer_head **bhp, + unsigned ahead, const char *msg) +{ + struct buffer_head *bh; + spadfs_cond_resched(); + if (likely((*bhp = bh = sb_find_get_block(fs->s, + secno >> fs->sectors_per_buffer_bits)) != NULL)) { + if (unlikely(!buffer_uptodate(bh))) { + __brelse(bh); + *bhp = NULL; + goto read_phys; + } + spadfs_track_buffer(fs, bh, msg); + return spadfs_buffer_data(fs, secno, bh); + } +read_phys: + return spadfs_read_sector_physical(fs, secno, bhp, ahead, msg); +} + +#ifdef SPADFS_DO_PREFETCH +#ifndef NEW_BIO +static void end_io_multibuffer_read(struct bio *bio, int err); +#else +static void end_io_multibuffer_read(struct bio *bio); +#endif +#endif + +static noinline void *spadfs_read_sector_physical(SPADFS *fs, sector_t secno, + struct buffer_head **bhp, + unsigned ahead, const char *msg) +{ + struct buffer_head *bh; + sector_t blockno; + if (unlikely(secno + ahead < secno) || + unlikely(secno + ahead >= fs->size)) { + ahead = 0; + if (unlikely(secno >= fs->size)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "access out of device, block %Lx, size %Lx at %s", + (unsigned long long)secno, + (unsigned long long)fs->size, + msg); + return ERR_PTR(-EFSERROR); + } + } + blockno = secno >> fs->sectors_per_buffer_bits; +#ifdef SPADFS_DO_PREFETCH + if (unlikely(ahead > BIO_MAX_VECS * PAGE_SIZE / 512)) + ahead = BIO_MAX_VECS * PAGE_SIZE / 512; + { + unsigned max_sec; + max_sec = queue_max_sectors(bdev_get_queue(fs->s->s_bdev)); + if (ahead > max_sec) { + ahead = max_sec; + if (unlikely(ahead & (ahead - 1))) + ahead = 1U << (fls(ahead) - 1); + } + } + ahead >>= fs->sectors_per_buffer_bits; + if (ahead > 1) { + struct bio *bio; + void **link; + sector_t reada_blockno = blockno; + unsigned i, bio_pages; + + if (likely(!(ahead & (ahead - 1))) && + likely(ahead > 1 << + (fs->sectors_per_page_bits - fs->sectors_per_buffer_bits))) + reada_blockno &= ~(sector_t)(ahead - 1); +retry_readahead: + bio_pages = (ahead >> (PAGE_SHIFT - 9 - fs->sectors_per_buffer_bits)); + bio_pages += fs->sectors_per_buffer_bits != PAGE_SHIFT - 9; + if (unlikely(bio_pages > BIO_MAX_VECS)) + bio_pages = BIO_MAX_VECS; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,18,0) + bio = bio_alloc(fs->s->s_bdev, bio_pages, REQ_OP_READ | REQ_RAHEAD | REQ_SYNC, (GFP_NOIO & ~__GFP_DIRECT_RECLAIM) | __GFP_NORETRY | __GFP_NOWARN); + if (unlikely(!bio)) + goto no_rdahead; +#else + bio = bio_alloc((GFP_NOIO & ~__GFP_DIRECT_RECLAIM) | __GFP_NORETRY | __GFP_NOWARN, bio_pages); + if (unlikely(!bio)) + goto no_rdahead; + bio_set_dev(bio, fs->s->s_bdev); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,8,0) + bio_set_op_attrs(bio, REQ_OP_READ, REQ_RAHEAD | REQ_SYNC); +#endif +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0) + bio->bi_sector +#else + bio->bi_iter.bi_sector +#endif + = reada_blockno << fs->sectors_per_buffer_bits; + bio->bi_end_io = end_io_multibuffer_read; + link = &bio->bi_private; + bio->bi_private = NULL; + for (i = 0; i < ahead; i++) { + struct buffer_head *bh; + bh = sb_getblk(fs->s, reada_blockno + i); + if (unlikely(!bh)) + break; +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) + if (unlikely(test_set_buffer_locked(bh))) +#else + if (unlikely(!trylock_buffer(bh))) +#endif + goto brelse_break; + if (unlikely(buffer_uptodate(bh))) + goto unlock_brelse_break; + if (unlikely(!bio_add_page(bio, bh->b_page, bh->b_size, + bh_offset(bh)))) { +unlock_brelse_break: + unlock_buffer(bh); +brelse_break: + __brelse(bh); + break; + } + *link = bh; + link = &bh->b_private; + bh->b_private = NULL; + } + if (likely(reada_blockno + i > blockno)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,8,0) + submit_bio(READA, bio); +#else + submit_bio(bio); +#endif + } else { +#ifndef NEW_BIO + bio_endio(bio, -EINTR); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,13,0) + bio->bi_error = -EINTR; + bio_endio(bio); +#else + bio->bi_status = BLK_STS_IOERR; + bio_endio(bio); +#endif + if (blockno != reada_blockno) { + if (likely(!(ahead & (ahead - 1)))) + ahead -= (blockno & (ahead - 1)); + if (likely(ahead > 1)) { + reada_blockno = blockno; + goto retry_readahead; + } + } + } + } +no_rdahead: +#endif +#if 0 + if (!bhp) + return NULL; +#endif + if (likely((*bhp = bh = sb_bread(fs->s, blockno)) != NULL)) { + spadfs_track_buffer(fs, bh, msg); + return spadfs_buffer_data(fs, secno, bh); + } + spadfs_error(fs, TXFLAGS_IO_READ_ERROR, "read error, block %Lx at %s", + (unsigned long long)secno, msg); + return ERR_PTR(-EIO); +} + +#ifdef SPADFS_DO_PREFETCH +#ifndef NEW_BIO +static void end_io_multibuffer_read(struct bio *bio, int err) +#else +static void end_io_multibuffer_read(struct bio *bio) +#endif +{ + struct buffer_head *bh = bio->bi_private; + while (bh) { + struct buffer_head *next_bh = bh->b_private; + bh->b_private = NULL; +#ifndef NEW_BIO + end_buffer_read_sync(bh, test_bit(BIO_UPTODATE, + &bio->bi_flags)); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,13,0) + end_buffer_read_sync(bh, !bio->bi_error); +#else + end_buffer_read_sync(bh, bio->bi_status == BLK_STS_OK); +#endif + bh = next_bh; + } + bio_put(bio); +} +#endif + +void *spadfs_get_new_sector(SPADFS *fs, sector_t secno, struct buffer_head **bhp, const char *msg) +{ + struct buffer_head *bh; + spadfs_cond_resched(); + if (unlikely(secno >= fs->size)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "creating block out of device, block %Lx, size %Lx at %s", + (unsigned long long)secno, + (unsigned long long)fs->size, + msg); + return ERR_PTR(-EFSERROR); + } + if (unlikely((unsigned long)secno & ((1 << fs->sectors_per_buffer_bits) - 1))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "unaligned get new sector %Lx at %s", + (unsigned long long)secno, + msg); + return ERR_PTR(-EFSERROR); + } + if (likely((*bhp = bh = sb_getblk(fs->s, secno >> fs->sectors_per_buffer_bits)) != NULL)) { + if (!buffer_uptodate(bh)) + wait_on_buffer(bh); + set_buffer_uptodate(bh); + spadfs_track_buffer(fs, bh, msg); + return bh->b_data; + } + spadfs_error(fs, TXFLAGS_IO_READ_ERROR, + "can't get new sector %Lx at %s", + (unsigned long long)secno, + msg); + return ERR_PTR(-ENOMEM); +} + +#if 0 +void spadfs_prefetch_sector(SPADFS *fs, sector_t secno, unsigned ahead, const char *msg) +{ +#ifdef SPADFS_DO_PREFETCH + struct buffer_head *bh; + if (likely((bh = sb_find_get_block(fs->s, + secno >> fs->sectors_per_buffer_bits)) != NULL)) { + __brelse(bh); + return; + } + spadfs_read_sector_physical(fs, secno, NULL, ahead, msg); +#endif +} +#endif + +void spadfs_discard_buffers(SPADFS *fs, sector_t start, sector_t n_sectors) +{ + start >>= fs->sectors_per_buffer_bits; + n_sectors >>= fs->sectors_per_buffer_bits; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) + for (; n_sectors; n_sectors--, start++) + bforget(sb_find_get_block(fs->s, start)); +#else + clean_bdev_aliases(fs->s->s_bdev, start, n_sectors); +#endif +} + +#ifdef CHECK_BUFFER_LEAKS + +struct tracked_buffer { + struct hlist_node list; + sector_t block; + const char *msg; +}; + +static void spadfs_track_buffer(SPADFS *fs, struct buffer_head *bh, + const char *msg) +{ + struct tracked_buffer *tb; + tb = kmalloc(sizeof(struct tracked_buffer), GFP_NOFS); + mutex_lock(&fs->buffer_track_lock); + if (likely(tb != NULL)) { + tb->block = bh->b_blocknr; + tb->msg = msg; + hlist_add_head(&tb->list, &fs->buffer_list); + } else { + fs->buffer_oom_events++; + } + mutex_unlock(&fs->buffer_track_lock); +} + +void spadfs_brelse(SPADFS *fs, struct buffer_head *bh) +{ + spadfs_drop_reference(fs, bh); +#ifdef CHECK_BUFFER_WRITES +#ifdef CHECK_BUFFER_WRITES_RANDOMIZE + if (!(random32() & 63)) +#endif + spadfs_sync_dirty_buffer(bh); +#endif + __brelse(bh); +} + +void spadfs_drop_reference(SPADFS *fs, struct buffer_head *bh) +{ + struct tracked_buffer *tb; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) + struct hlist_node *n; +#endif + mutex_lock(&fs->buffer_track_lock); +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) + hlist_for_each_entry(tb, n, &fs->buffer_list, list) +#else + hlist_for_each_entry(tb, &fs->buffer_list, list) +#endif + { + if (tb->block == bh->b_blocknr) { + hlist_del(&tb->list); + kfree(tb); + goto found; + } + spadfs_cond_resched(); + } + BUG_ON(!fs->buffer_oom_events); + fs->buffer_oom_events--; +found: + mutex_unlock(&fs->buffer_track_lock); +} + +void spadfs_buffer_leaks_init(SPADFS *fs) +{ + mutex_init(&fs->buffer_track_lock); + INIT_HLIST_HEAD(&fs->buffer_list); + fs->buffer_oom_events = 0; +} + +void spadfs_buffer_leaks_done(SPADFS *fs) +{ + while (unlikely(!hlist_empty(&fs->buffer_list))) { + struct tracked_buffer *tb = list_entry(fs->buffer_list.first, + struct tracked_buffer, list); + printk(KERN_ERR "spadfs internal error: buffer %Lx leaked at %s\n", + (unsigned long long)tb->block, tb->msg); + hlist_del(&tb->list); + kfree(tb); + } + if (unlikely(fs->buffer_oom_events != 0)) + printk(KERN_ERR "spadfs internal error: %lx unknown buffer leaked\n", + fs->buffer_oom_events); + mutex_destroy(&fs->buffer_track_lock); +} + +#endif diff --git a/fs/spadfs/bufstruc.c b/fs/spadfs/bufstruc.c new file mode 100644 index 000000000..3e952a128 --- /dev/null +++ b/fs/spadfs/bufstruc.c @@ -0,0 +1,161 @@ +#include "spadfs.h" + +/* + * Read various filesystem structures, and check for magic and checksum + * TODO: + * check checksum only when reading from disk. + * use some flag that is reset when reading buffer from disk (which?) + */ + +struct txblock *spadfs_read_tx_block(SPADFS *fs, struct buffer_head **bhp, + const char *msg) +{ + struct txblock *tb = spadfs_read_sector(fs, fs->txb_sec, bhp, 0, msg); + if (unlikely(IS_ERR(tb))) return tb; + if (unlikely(SPAD2CPU32_LV(&tb->magic) != + spadfs_magic(fs, fs->txb_sec, TXBLOCK_MAGIC))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "bad magic on tx block %Lx at %s", + (unsigned long long)fs->txb_sec, + msg); +rel_err: + spadfs_brelse(fs, *bhp); + return ERR_PTR(-EFSERROR); + } + if (unlikely(check_checksums(fs))) { + if (unlikely(__byte_sum(tb, 512) != CHECKSUM_BASE)) { + spadfs_error(fs, TXFLAGS_CHECKSUM_ERROR, + "bad checksum on tx block %Lx at %s", + (unsigned long long)fs->txb_sec, + msg); + goto rel_err; + } + } + return tb; +} + +/* + * struct_type means what can we accept: + * SRFB_FNODE : fnode_block + * SRFB_DNODE : dnode + * SRFB_FIXED_FNODE : fixed fnode block + * if more are set, the type of returned structure must be determined by + * the caller from magic. + */ + +struct fnode_block *spadfs_read_fnode_block(SPADFS *fs, sector_t secno, + struct buffer_head **bhp, + int struct_type, const char *msg) +{ + struct fnode_block *fnode_block = spadfs_read_sector(fs, secno, bhp, + fs->metadata_prefetch , msg); + if (unlikely(IS_ERR(fnode_block))) + return fnode_block; + + if (likely(SPAD2CPU32_LV(&fnode_block->magic) == + spadfs_magic(fs, secno, FNODE_BLOCK_MAGIC))) { + if (unlikely(!(struct_type & SRFB_FNODE))) + goto bad_magic; +check_fn: + if (unlikely(check_checksums(fs)) && + likely(fnode_block->flags & FNODE_BLOCK_CHECKSUM_VALID)) { + start_concurrent_atomic_buffer_read(fs, *bhp); + if (likely(fnode_block->flags & + FNODE_BLOCK_CHECKSUM_VALID)) { + if (unlikely(__byte_sum(fnode_block, 512) != + CHECKSUM_BASE)) { + end_concurrent_atomic_buffer_read(fs, + *bhp); + spadfs_error(fs, TXFLAGS_CHECKSUM_ERROR, + "bad checksum on fnode block %Lx at %s", + (unsigned long long)secno, + msg); + goto rel_err; + } + } + end_concurrent_atomic_buffer_read(fs, *bhp); + } + return fnode_block; + } + if (likely(SPAD2CPU32_LV(&fnode_block->magic) == + spadfs_magic(fs, secno, DNODE_PAGE_MAGIC))) { + if (unlikely(!(struct_type & SRFB_DNODE))) + goto bad_magic; + if (unlikely((((struct dnode_page *)fnode_block)->gflags & + DNODE_GFLAGS_PAGE_SIZE_BITS_MINUS_1) >> + DNODE_GFLAGS_PAGE_SIZE_BITS_MINUS_1_SHIFT != + fs->sectors_per_page_bits - 1)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "dnode %Lx has invalid page size at %s", + (unsigned long long)secno, + msg); + goto rel_err; + } + return fnode_block; + } + if (likely(SPAD2CPU32_LV(&fnode_block->magic) == + spadfs_magic(fs, secno, FIXED_FNODE_BLOCK_MAGIC))) { + if (unlikely(!(struct_type & SRFB_FIXED_FNODE))) + goto bad_magic; + goto check_fn; + } +bad_magic: + spadfs_error(fs, TXFLAGS_FS_ERROR, + "bad magic %08x on fnode block %Lx at %s", + SPAD2CPU32(fnode_block->magic), + (unsigned long long)secno, + msg); +rel_err: + spadfs_brelse(fs, *bhp); + return ERR_PTR(-EFSERROR); +} + +struct anode *spadfs_read_anode(SPADFS *fs, sector_t secno, + struct buffer_head **bhp, unsigned *vx, + int read_lock, const char *msg) +{ + struct anode *anode = spadfs_read_sector(fs, secno, bhp, + 1 << fs->sectors_per_page_bits, msg); + if (unlikely(IS_ERR(anode))) + return anode; + + if (read_lock) + start_concurrent_atomic_buffer_read(fs, *bhp); + + if (unlikely(SPAD2CPU32_LV(&anode->magic) != + spadfs_magic(fs, secno, ANODE_MAGIC))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "bad magic on anode %Lx at %s", + (unsigned long long)secno, + msg); + goto rel_err; + } + if (unlikely(check_checksums(fs)) && + likely(anode->flags & ANODE_CHECKSUM_VALID)) { + if (unlikely(__byte_sum(anode, 512) != CHECKSUM_BASE)) { + spadfs_error(fs, TXFLAGS_CHECKSUM_ERROR, + "bad checksum on anode %Lx at %s", + (unsigned long long)secno, + msg); + goto rel_err; + } + } + *vx = anode->valid_extents; + if (unlikely((unsigned)(*vx - 1) >= ANODE_N_EXTENTS)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "anode %Lx has %u valid extents at %s", + (unsigned long long)secno, + (unsigned)anode->valid_extents, + msg); + goto rel_err; + } + return anode; + +rel_err: + if (read_lock) + end_concurrent_atomic_buffer_read(fs, *bhp); + + spadfs_brelse(fs, *bhp); + return ERR_PTR(-EFSERROR); +} + diff --git a/fs/spadfs/dir.c b/fs/spadfs/dir.c new file mode 100644 index 000000000..eb63c7088 --- /dev/null +++ b/fs/spadfs/dir.c @@ -0,0 +1,2188 @@ +#include "spadfs.h" +#include "dir.h" + +void really_do_fnode_block_checksum(struct fnode_block *fnode_block) +{ + fnode_block->flags |= FNODE_BLOCK_CHECKSUM_VALID; + fnode_block->checksum ^= CHECKSUM_BASE ^ + __byte_sum(fnode_block, FNODE_BLOCK_SIZE); +} + +static int spadfs_access_dnode(SPADFS *fs, sector_t secno, unsigned version, + unsigned i, sector_t *result, int wr) +{ + struct buffer_head *bh; + char *data; + i = (i * DNODE_PAGE_ENTRY_SIZE) + DNODE_ENTRY_OFFSET; + if (version) + i += fs->dnode_data_size; + + data = spadfs_read_sector(fs, secno + (i >> 9), &bh, 0, + "spadfs_access_dnode 1"); + if (unlikely(IS_ERR(data))) + return PTR_ERR(data); + + data += i & 511; + if (likely((i & 511) <= 512 - DNODE_PAGE_ENTRY_SIZE)) { + if (likely(!wr)) { + *result = MAKE_D_OFF3( + ((struct dnode_page_entry *)data)->b0, + ((struct dnode_page_entry *)data)->b1, + ((struct dnode_page_entry *)data)->b2); + } else { + ((struct dnode_page_entry *)data)->b0 = + MAKE_PART_30(*result); + ((struct dnode_page_entry *)data)->b1 = + MAKE_PART_31(*result); + ((struct dnode_page_entry *)data)->b2 = + MAKE_PART_32(*result); + } + } else { + struct dnode_page_entry dp; + if (likely(!wr)) + memcpy(&dp, data, 512 - (i & 511)); + else { + dp.b0 = MAKE_PART_30(*result); + dp.b1 = MAKE_PART_31(*result); + dp.b2 = MAKE_PART_32(*result); + memcpy(data, &dp, 512 - (i & 511)); + mark_buffer_dirty(bh); + } + spadfs_brelse(fs, bh); + + data = spadfs_read_sector(fs, secno + (i >> 9) + 1, &bh, 0, + "spadfs_access_dnode 2"); + if (unlikely(IS_ERR(data))) + return PTR_ERR(data); + + if (likely(!wr)) { + memcpy((char *)&dp + (512 - (i & 511)), data, + DNODE_PAGE_ENTRY_SIZE - (512 - (i & 511))); + *result = MAKE_D_OFF3(dp.b0, dp.b1, dp.b2); + } else + memcpy(data, (char *)&dp + (512 - (i & 511)), + DNODE_PAGE_ENTRY_SIZE - (512 - (i & 511))); + } + + if (wr) + mark_buffer_dirty(bh); + + spadfs_brelse(fs, bh); + return 0; +} + +static int spadfs_read_dnode(SPADFS *fs, sector_t secno, unsigned version, + unsigned i, sector_t *result) +{ + return spadfs_access_dnode(fs, secno, version, i, result, 0); +} + +static int spadfs_write_dnode(SPADFS *fs, sector_t secno, unsigned version, + unsigned i, sector_t result) +{ + return spadfs_access_dnode(fs, secno, version, i, &result, 1); +} + +static int dnode_version(SPADFS *fs, struct dnode_page *d) +{ + return !CC_VALID(fs, &d->cc, &d->txc); +} + +static int spadfs_begin_modify_dnode(SPADFS *fs, sector_t dno) +{ + int version; + struct buffer_head *bh; + struct dnode_page *d; + + d = (struct dnode_page *)spadfs_read_fnode_block(fs, dno, &bh, + SRFB_DNODE, "spadfs_begin_modify_dnode 1"); + if (unlikely(IS_ERR(d))) + return PTR_ERR(d); + + version = dnode_version(fs, d); + + if (CC_CURRENT(fs, &d->cc, &d->txc)) { + d->flags[version] &= ~DNODE_CHECKSUM_VALID; + mark_buffer_dirty(bh); + spadfs_brelse(fs, bh); + return version; + } else { + unsigned from, to; + char *datafrom, *datato; + struct buffer_head *bhfrom, *bhto; + unsigned remaining, len; + + start_atomic_buffer_modify(fs, bh); + d->flags[version ^ 1] &= ~DNODE_CHECKSUM_VALID; + CC_SET_CURRENT(fs, &d->cc, &d->txc); + end_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + + from = DNODE_ENTRY_OFFSET + version * fs->dnode_data_size; + to = DNODE_ENTRY_OFFSET + (version ^ 1) * fs->dnode_data_size; + remaining = fs->dnode_data_size; + datafrom = spadfs_read_sector(fs, dno + (from >> 9), &bhfrom, 0, + "spadfs_begin_modify_dnode 2"); + if (unlikely(IS_ERR(datafrom))) + return PTR_ERR(datafrom); + + datafrom += from & 511; +reread_to: + datato = spadfs_read_sector(fs, dno + (to >> 9), &bhto, 0, + "spadfs_begin_modify_dnode 3"); + if (unlikely(IS_ERR(datato))) { + spadfs_brelse(fs, bhfrom); + return PTR_ERR(datato); + } + datato += to & 511; + +copy_again: + len = 512 - (from & 511); + if (512 - (to & 511) < len) + len = 512 - (to & 511); + if (unlikely(remaining < len)) + len = remaining; + memcpy(datato, datafrom, len); + datafrom += len; + datato += len; + from += len; + to += len; + remaining -= len; + if (remaining) { + if (!(from & 511)) { + spadfs_brelse(fs, bhfrom); + datafrom = spadfs_read_sector(fs, + dno + (from >> 9), + &bhfrom, 0, + "spadfs_begin_modify_dnode 4"); + if (unlikely(IS_ERR(datafrom))) { + spadfs_brelse(fs, bhto); + return PTR_ERR(datafrom); + } + datafrom += from & 511; + } + if (!(to & 511)) { + mark_buffer_dirty(bhto); + spadfs_brelse(fs, bhto); + goto reread_to; + } + if (unlikely(from & 511)) + panic("spadfs: unaligned from pointer: %d, %d", + from, to); + goto copy_again; + } + spadfs_brelse(fs, bhfrom); + mark_buffer_dirty(bhto); + spadfs_brelse(fs, bhto); + return version ^ 1; + } +} + +int spadfs_alloc_fixed_fnode_block(SPADFNODE *fn, sector_t hint, unsigned size, + u16 hint_small, u16 hint_big, + sector_t *result) +{ + SPADFS *fs = fn->fs; + int r; + struct fixed_fnode_block *fx; + struct alloc al; + struct buffer_head *bh; + + al.sector = hint; + al.n_sectors = 1U << fs->sectors_per_disk_block_bits; + al.extra_sectors = 0; + al.flags = ALLOC_METADATA; + al.reservation = NULL; + r = spadfs_alloc_blocks(fs, &al); + if (unlikely(r)) + return r; + *result = al.sector; + + fx = spadfs_get_new_sector(fs, al.sector, &bh, "spadfs_alloc_fixed_fnode_block"); + if (unlikely(IS_ERR(fx))) + return PTR_ERR(fx); + memset(fx, 0, 512U << fs->sectors_per_buffer_bits); + CPU2SPAD32_LV(&fx->magic, spadfs_magic(fs, al.sector, FIXED_FNODE_BLOCK_MAGIC)); + fx->flags = 0; + /* + * cc/txc doesn't matter because this block won't be used in case of + * crash. Set to current (cc, txc) to prevent copy on next update. + */ + CPU2SPAD16_LV(&fx->cc, fs->cc); + CPU2SPAD32_LV(&fx->txc, fs->txc); + CPU2SPAD16_LV(&fx->hint_small, hint_small); + CPU2SPAD16_LV(&fx->hint_large, hint_big); + CPU2SPAD64_LV(&fx->nlink0, 1); +#define fnode ((struct fnode *)fx->fnode0) + CPU2SPAD16_LV(&fnode->next, size); +#undef fnode + do_fnode_block_checksum(fs, (struct fnode_block *)fx); + + mark_buffer_dirty(bh); + spadfs_brelse(fs, bh); + + return 0; +} + +static int spadfs_account_directory_blocks(SPADFNODE *fn, unsigned n_sectors) +{ + int r; + loff_t old_i_size, new_i_size; + + if (likely(directory_size(fn->fs))) { + + old_i_size = inode(fn)->i_size; + new_i_size = old_i_size + n_sectors * 512; + + if (unlikely(new_i_size < old_i_size)) { + spadfs_error(fn->fs, TXFLAGS_FS_ERROR, + "directory size overflow, size %Lx, adding %x", + (unsigned long long)old_i_size, + n_sectors * 512); + new_i_size = 0; + } + +#ifdef SPADFS_QUOTA + r = dquot_alloc_space_nodirty(inode(fn), + new_i_size - old_i_size); + if (unlikely(r)) + goto ret_r; +#else + inode_add_bytes(inode(fn), new_i_size - old_i_size); +#endif + + i_size_write(inode(fn), new_i_size); + + r = spadfs_write_directory(fn); + if (unlikely(r)) { + i_size_write(inode(fn), old_i_size); + goto unaccount_ret_r; + } + } + return 0; + +unaccount_ret_r: +#ifdef SPADFS_QUOTA + dquot_free_space_nodirty(inode(fn), new_i_size - old_i_size); +#else + inode_sub_bytes(inode(fn), new_i_size - old_i_size); +#endif + +#ifdef SPADFS_QUOTA +ret_r: +#endif + return r; +} + +static int spadfs_free_directory_blocks(SPADFNODE *fn, sector_t start, + unsigned n_sectors) +{ + int r; + + if (likely(directory_size(fn->fs))) { + + loff_t old_i_size, new_i_size; + + old_i_size = inode(fn)->i_size; + + if (likely(old_i_size >= n_sectors * 512)) + new_i_size = old_i_size - n_sectors * 512; + else { + spadfs_error(fn->fs, TXFLAGS_FS_ERROR, + "directory size miscounted, size %Lx, freeing %x", + (unsigned long long)old_i_size, + n_sectors * 512); + + new_i_size = 0; + } + + +#ifdef SPADFS_QUOTA + dquot_free_space_nodirty(inode(fn), old_i_size - new_i_size); +#else + inode_sub_bytes(inode(fn), old_i_size - new_i_size); +#endif + + i_size_write(inode(fn), new_i_size); + + r = spadfs_write_directory(fn); + if (unlikely(r)) + return r; + } + return spadfs_free_blocks_metadata(fn->fs, start, n_sectors); +} + +int spadfs_alloc_leaf_page(SPADFNODE *fn, sector_t hint, unsigned n_sectors, + sector_t prev, sector_t *result, int noaccount) +{ + SPADFS *fs = fn->fs; + int r; + unsigned i, j; + struct alloc al; + + al.sector = hint; + al.n_sectors = n_sectors; + al.extra_sectors = 0; + al.flags = ALLOC_METADATA; + al.reservation = NULL; + r = spadfs_alloc_blocks(fs, &al); + if (unlikely(r)) + return r; + *result = al.sector; + + for (i = 0; i < n_sectors; i += 1U << fs->sectors_per_buffer_bits) { + struct buffer_head *bh; + struct fnode_block *b = spadfs_get_new_sector(fs, al.sector + i, &bh, "spadfs_alloc_leaf_page"); + if (unlikely(IS_ERR(b))) { + r = PTR_ERR(b); + goto free_return_r; + } + + for (j = 0; j < 1U << fs->sectors_per_buffer_bits; + j++, b = (struct fnode_block *)((char *)b + FNODE_BLOCK_SIZE)) { + struct fnode *fn; + memset(b, 0, FNODE_BLOCK_SIZE); + if (!i && !j) { + b->prev0 = MAKE_PART_0(prev); + b->prev1 = MAKE_PART_1(prev); + } + CPU2SPAD32_LV(&b->magic, spadfs_magic(fs, al.sector + (i + j), FNODE_BLOCK_MAGIC)); + b->flags = (FNODE_BLOCK_FIRST * (!i && !j)) | + (FNODE_BLOCK_LAST * (i + j == n_sectors - 1)); + CPU2SPAD32_LV(&b->txc, TXC_INVALID(0)); + CPU2SPAD16_LV(&b->cc, 0); + fn = b->fnodes; + CPU2SPAD16_LV(&fn->next, (FNODE_MAX_SIZE & FNODE_NEXT_SIZE) | FNODE_NEXT_FREE); + CPU2SPAD16_LV(&fn->cc, 0); + CPU2SPAD32_LV(&fn->txc, 0); + do_fnode_block_checksum(fs, b); + } + mark_buffer_dirty(bh); + spadfs_brelse(fs, bh); + } + if (!noaccount) { + r = spadfs_account_directory_blocks(fn, n_sectors); + if (unlikely(r)) + goto free_return_r; + } + return 0; + +free_return_r: + spadfs_free_blocks_metadata(fs, al.sector, n_sectors); + return r; +} + +static int spadfs_alloc_dnode_page(SPADFNODE *fn, sector_t hint, + sector_t *result, sector_t parent, + sector_t ptr_init) +{ + SPADFS *fs = fn->fs; + int r; + unsigned i; + struct alloc al; + + *result = 0; /* avoid warning */ + + al.sector = hint; + al.n_sectors = fs->dnode_page_sectors; + al.extra_sectors = 0; + al.flags = ALLOC_METADATA; + al.reservation = NULL; + r = spadfs_alloc_blocks(fs, &al); + if (unlikely(r)) + return r; + *result = al.sector; + + for (i = 0; i < fs->dnode_page_sectors; i += 1U << fs->sectors_per_buffer_bits) { + struct buffer_head *bh; + struct dnode_page *d = spadfs_get_new_sector(fs, al.sector + i, &bh, "spadfs_alloc_dnode_page"); + if (unlikely(IS_ERR(d))) { + r = PTR_ERR(d); + goto free_return_r; + } + + memset(d, 0, 512U << fs->sectors_per_buffer_bits); + if (!i) { + CPU2SPAD32_LV(&d->magic, spadfs_magic(fs, al.sector, DNODE_PAGE_MAGIC)); + d->gflags = (fs->sectors_per_page_bits - 1) << DNODE_GFLAGS_PAGE_SIZE_BITS_MINUS_1_SHIFT; + CPU2SPAD64_LV(&d->up_dnode, parent); + CPU2SPAD32_LV(&d->txc, fs->txc); + CPU2SPAD16_LV(&d->cc, fs->cc); + } + mark_buffer_dirty(bh); + spadfs_brelse(fs, bh); + } + for (i = 0; i < 1U << fs->dnode_hash_bits; i++) { + r = spadfs_write_dnode(fs, al.sector, 0, i, ptr_init); + if (unlikely(r)) + goto free_return_r; + } + + r = spadfs_account_directory_blocks(fn, fs->dnode_page_sectors); + if (unlikely(r)) + goto free_return_r; + + return 0; + +free_return_r: + spadfs_free_blocks_metadata(fs, al.sector, fs->dnode_page_sectors); + return r; +} + +struct fnode_block *spadfs_find_hash_block(SPADFNODE *f, hash_t hash, + struct buffer_head **bhp, + sector_t *secno, hash_t *next_hash) +{ + sector_t old_secno; + unsigned hash_bits; + unsigned hpos; + unsigned version; + struct fnode_block *fnode_block; + int r; + sector_t c[2]; + c[1] = 0; + *secno = f->root; + hash_bits = 0; + + if (unlikely(next_hash != NULL)) + *next_hash = 0; +again: + if (unlikely(spadfs_stop_cycles(f->fs, *secno, &c, + "spadfs_find_hash_block"))) + return ERR_PTR(-EFSERROR); + + fnode_block = spadfs_read_fnode_block(f->fs, *secno, bhp, + SRFB_FNODE | SRFB_DNODE, + "spadfs_find_hash_block"); + if (unlikely(IS_ERR(fnode_block))) + return fnode_block; + + if (likely(SPAD2CPU32_LV(&fnode_block->magic) == + spadfs_magic(f->fs, *secno, FNODE_BLOCK_MAGIC))) + return fnode_block; + + version = dnode_version(f->fs, (struct dnode_page *)fnode_block); + spadfs_brelse(f->fs, *bhp); + + if (unlikely(hash_bits >= SPADFS_HASH_BITS)) { + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "too deep dnode tree, started at %Lx, looked up hash %08Lx", + (unsigned long long)f->root, + (unsigned long long)hash); + return ERR_PTR(-EFSERROR); + } + hpos = (hash >> hash_bits) & ((1 << f->fs->dnode_hash_bits) - 1); + old_secno = *secno; + r = spadfs_read_dnode(f->fs, *secno, version, hpos, secno); + if (unlikely(r)) + return ERR_PTR(r); + if (unlikely(next_hash != NULL)) { + int i; + for (i = f->fs->dnode_hash_bits - 1; i >= 0; i--) + if (!(hpos & (1 << i)) && + likely(i + hash_bits < SPADFS_HASH_BITS)) { + sector_t test_secno; + r = spadfs_read_dnode(f->fs, old_secno, version, + (hpos & ((1 << i) - 1)) | (1 << i), + &test_secno); + if (unlikely(r)) + return ERR_PTR(r); + if (unlikely(!test_secno) || + test_secno != *secno) { +#if 0 + if (test_secno != 0) { + spadfs_prefetch_sector(f->fs, test_secno, f->fs->metadata_prefetch, "spadfs_find_hash_block"); + } +#endif + *next_hash = + (hash & (((hash_t)1 << + (hash_bits + i)) - 1)) | + (hash_t)1 << (hash_bits + i); + break; + } + } + } + hash_bits += f->fs->dnode_hash_bits; + if (unlikely(!*secno)) + return NULL; + goto again; +} + +/* + * Directory entry sector/position is returned in ino. + * If for_delete is set, ino is input argument too (pointer to fixed fnode + * whose entry to delete) --- the reason for it is that during rename + * there are temporarily two entries with the same name in the directory. + */ + +int spadfs_lookup_ino(SPADFNODE *f, struct qstr *qstr, spadfs_ino_t *ino, + int for_delete) +{ + struct buffer_head *bh; + struct fnode_block *fnode_block; + struct fnode *fnode; + sector_t secno; + unsigned size, namelen; + sector_t c[2]; + c[1] = 0; + + fnode_block = spadfs_find_hash_block(f, + name_len_hash((const char *)qstr->name, qstr->len), &bh, &secno, NULL); + if (unlikely(!fnode_block)) + return 1; + +next_fnode_block: + if (unlikely(IS_ERR(fnode_block))) + return PTR_ERR(fnode_block); + fnode = fnode_block->fnodes; + +next_fnode: + VALIDATE_FNODE(f->fs, fnode_block, fnode, size, namelen, + ok, skip, bad_fnode); + +ok: + if (!spadfs_compare_names(f->fs, (const char *)qstr->name, qstr->len, + FNODE_NAME(fnode), namelen)) { + if (unlikely(for_delete)) { + if (!(fnode->flags & FNODE_FLAGS_HARDLINK)) + goto skip; + if (unlikely(*ino != make_fixed_spadfs_ino_t( + MAKE_D_OFF(fnode->anode0, fnode->anode1)))) + goto skip; + goto set_ino_to_entry; + } + if (likely(!(fnode->flags & FNODE_FLAGS_HARDLINK))) +set_ino_to_entry: + *ino = make_spadfs_ino_t(secno, + (char *)fnode - (char *)fnode_block); + else + *ino = make_fixed_spadfs_ino_t( + MAKE_D_OFF(fnode->anode0, fnode->anode1)); + spadfs_brelse(f->fs, bh); + return 0; + } + +skip: + fnode = (struct fnode *)((char *)fnode + size); + if (likely(((unsigned long)fnode & (FNODE_BLOCK_SIZE - 1)) != 0)) + goto next_fnode; + + if (!(fnode_block->flags & FNODE_BLOCK_LAST)) { + spadfs_brelse(f->fs, bh); + secno++; +read_next_fnode_block: + if (unlikely(spadfs_stop_cycles(f->fs, secno, &c, + "spadfs_lookup_ino"))) + return -EFSERROR; + fnode_block = spadfs_read_fnode_block(f->fs, secno, &bh, + SRFB_FNODE, + "spadfs_lookup_ino"); + goto next_fnode_block; + } + + if (unlikely(CC_VALID(f->fs, &fnode_block->cc, &fnode_block->txc))) { + secno = MAKE_D_OFF(fnode_block->next0, fnode_block->next1); + spadfs_brelse(f->fs, bh); + goto read_next_fnode_block; + } + spadfs_brelse(f->fs, bh); + return 1; + +bad_fnode: + spadfs_brelse(f->fs, bh); + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "lookup: bad fnode on block %Lx", + (unsigned long long)secno); + return -EFSERROR; +} + +int spadfs_check_directory_empty(SPADFNODE *f) +{ + struct fnode_block *fnode_block; + struct buffer_head *bh; + sector_t secno; + struct fnode *fnode; + unsigned size, namelen; + hash_t hash = 0; + hash_t next_hash; + sector_t c[2]; + c[1] = 0; + +new_hash_lookup: + fnode_block = spadfs_find_hash_block(f, hash, &bh, &secno, &next_hash); + if (unlikely(!fnode_block)) + goto use_next_hash; + +next_fnode_block: + if (unlikely(IS_ERR(fnode_block))) + return PTR_ERR(fnode_block); + fnode = fnode_block->fnodes; + +next_fnode: + VALIDATE_FNODE(f->fs, fnode_block, fnode, size, namelen, + ok, skip, bad_fnode); + +ok: + spadfs_brelse(f->fs, bh); + return -ENOTEMPTY; + +skip: + fnode = (struct fnode *)((char *)fnode + size); + if (likely(((unsigned long)fnode & (FNODE_BLOCK_SIZE - 1)) != 0)) + goto next_fnode; + + if (!(fnode_block->flags & FNODE_BLOCK_LAST)) { + spadfs_brelse(f->fs, bh); + secno++; +read_next_fnode_block: + if (unlikely(spadfs_stop_cycles(f->fs, secno, &c, + "spadfs_check_directory_empty"))) { + return -EFSERROR; + } + fnode_block = spadfs_read_fnode_block(f->fs, secno, &bh, + SRFB_FNODE, + "spadfs_check_directory_empty"); + goto next_fnode_block; + } + + if (unlikely(CC_VALID(f->fs, &fnode_block->cc, &fnode_block->txc))) { + secno = MAKE_D_OFF(fnode_block->next0, fnode_block->next1); + spadfs_brelse(f->fs, bh); + goto read_next_fnode_block; + } + spadfs_brelse(f->fs, bh); + +use_next_hash: + if (unlikely(next_hash != 0)) { + hash = next_hash; + goto new_hash_lookup; + } + return 0; + +bad_fnode: + spadfs_brelse(f->fs, bh); + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "spadfs_check_directory_empty: bad fnode on block %Lx", + (unsigned long long)secno); + return -EFSERROR; +} + +static int spadfs_remove_recursive(SPADFNODE *fn, sector_t root, int depth) +{ + SPADFS *fs = fn->fs; + struct fnode_block *fnode_block; + struct buffer_head *bh; + sector_t c[2]; + int r; + c[1] = 0; + +next_in_chain: + fnode_block = spadfs_read_fnode_block(fs, root, &bh, + SRFB_FNODE | SRFB_DNODE, + "spadfs_remove_recursive 1"); + if (unlikely(IS_ERR(fnode_block))) + return PTR_ERR(fnode_block); + + if (likely(SPAD2CPU32_LV(&fnode_block->magic) == + spadfs_magic(fs, root, FNODE_BLOCK_MAGIC))) { + sector_t next_root; + unsigned nsec = 1; + while (!(fnode_block->flags & FNODE_BLOCK_LAST)) { + spadfs_brelse(fs, bh); + fnode_block = spadfs_read_fnode_block(fs, root + nsec, + &bh, SRFB_FNODE, + "spadfs_remove_recursive 2"); + if (unlikely(IS_ERR(fnode_block))) + return PTR_ERR(fnode_block); + + nsec++; + if (unlikely(nsec > 1U << fs->sectors_per_page_bits)) { + spadfs_brelse(fs, bh); + spadfs_error(fs, TXFLAGS_FS_ERROR, + "too long fnode block run at %Lx", + (unsigned long long)root); + return -EFSERROR; + } + } + next_root = 0; + if (CC_VALID(fs, &fnode_block->cc, &fnode_block->txc)) + next_root = MAKE_D_OFF(fnode_block->next0, + fnode_block->next1); + spadfs_brelse(fs, bh); + r = spadfs_free_directory_blocks(fn, root, nsec); + if (unlikely(r)) + return r; + + if (next_root) { + if (unlikely(spadfs_stop_cycles(fs, next_root, &c, + "spadfs_remove_recursive"))) + return -EFSERROR; + root = next_root; + goto next_in_chain; + } + return 0; + } else { + unsigned hp, i; + sector_t ln; + int version; + if (unlikely(depth >= SPADFS_HASH_BITS)) { + spadfs_brelse(fs, bh); + spadfs_error(fs, TXFLAGS_FS_ERROR, + "recurse overflow on block %Lx", + (unsigned long long)root); + return -EFSERROR; + } + version = dnode_version(fs, (struct dnode_page *)fnode_block); + spadfs_brelse(fs, bh); + ln = 0; + hp = 0; + while (1) { + sector_t n = 0; /* against warning */ + int r = spadfs_read_dnode(fs, root, version, hp, &n); + if (unlikely(r)) + return r; + if (n && n != ln) { + ln = n; + r = spadfs_remove_recursive(fn, n, + depth + fs->dnode_hash_bits); + if (unlikely(r)) + return r; + } + i = 1 << (fs->dnode_hash_bits - 1); + while (!((hp ^= i) & i)) + if (unlikely(!(i >>= 1))) + goto brk; + } +brk: + return spadfs_free_directory_blocks(fn, root, + fs->dnode_page_sectors); + } +} + +int spadfs_remove_directory(SPADFNODE *fn) +{ + int r; + r = spadfs_remove_recursive(fn, fn->root, 0); + if (unlikely(r)) + return r; + + if (likely(directory_size(fn->fs))) { + if (unlikely(inode(fn)->i_size != 0)) + spadfs_error(fn->fs, TXFLAGS_FS_ERROR, + "directory size miscounted, %Lx leaked", + (unsigned long long)inode(fn)->i_size); + } + + return 0; +} + +sector_t spadfs_alloc_hint(SPADFNODE *f, int hint) +{ + SPADFS *fs = f->fs; + struct fnode_block *fnode_block; + struct fnode *fnode; + struct buffer_head *bh; + unsigned group; + if (unlikely(is_deleted_file(f))) { +return_default: + if (unlikely(hint == HINT_META)) + return 0; + return (sector_t)fs->zones[1 + (hint == HINT_BIG)].grp_start << + fs->sectors_per_group_bits; + } + if (unlikely(hint == HINT_META)) + return f->fnode_block; + + if (unlikely(is_fnode_fixed(f))) { + fnode_block = spadfs_read_fnode_block(fs, f->fnode_block, &bh, + SRFB_FIXED_FNODE, + "spadfs_alloc_hint (fixed)"); + if (unlikely(IS_ERR(fnode_block))) + goto return_default; + if (hint == HINT_SMALL) + group = SPAD2CPU16_LV(&((struct fixed_fnode_block *) + fnode_block)->hint_small); + else + group = SPAD2CPU16_LV(&((struct fixed_fnode_block *) + fnode_block)->hint_large); + goto brelse_ret; + } + fnode_block = spadfs_read_fnode_block(fs, f->parent_fnode_block, &bh, + SRFB_FNODE | SRFB_FIXED_FNODE, + "spadfs_alloc_hint"); + if (unlikely(IS_ERR(fnode_block))) + goto return_default; + fnode = (struct fnode *)((char *)fnode_block + f->parent_fnode_pos); + if (hint == HINT_SMALL) + group = SPAD2CPU16_LV(&fnode->run1n); + else + group = SPAD2CPU16_LV(&fnode->run2n); + +brelse_ret: + spadfs_brelse(fs, bh); + return (sector_t)group << fs->sectors_per_group_bits; +} + +void spadfs_set_new_hint(SPADFNODE *f, struct alloc *al) +{ + struct fnode_block *fnode_block; + struct fnode *fnode; + struct buffer_head *bh; + unsigned hint; + SPADFS *fs = f->fs; + + if (unlikely(is_deleted_file(f))) + return; + + hint = al->sector >> fs->sectors_per_group_bits; + if (unlikely(is_fnode_fixed(f))) { + fnode_block = spadfs_read_fnode_block(fs, f->fnode_block, &bh, + SRFB_FIXED_FNODE, + "spadfs_set_new_hint (fixed)"); + if (unlikely(IS_ERR(fnode_block))) + return; + + start_concurrent_atomic_buffer_modify(fs, bh); + if (likely(al->flags & ALLOC_BIG_FILE)) + CPU2SPAD16_LV(&((struct fixed_fnode_block *)fnode_block) + ->hint_large, hint); + else + CPU2SPAD16_LV(&((struct fixed_fnode_block *)fnode_block) + ->hint_small, hint); + goto checksum_done; + } + + fnode_block = spadfs_read_fnode_block(fs, f->parent_fnode_block, &bh, + SRFB_FNODE | SRFB_FIXED_FNODE, + "spadfs_set_new_hint"); + if (unlikely(IS_ERR(fnode_block))) + return; + + fnode = (struct fnode *)((char *)fnode_block + f->parent_fnode_pos); + start_concurrent_atomic_buffer_modify(fs, bh); + if (likely(al->flags & ALLOC_BIG_FILE)) + CPU2SPAD16_LV(&fnode->run2n, hint); + else + CPU2SPAD16_LV(&fnode->run1n, hint); +checksum_done: + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); +} + +void spadfs_get_dir_hint(SPADFNODE *f, u16 *small, u16 *big) +{ + SPADFS *fs = f->fs; + struct fnode_block *fnode_block; + struct fnode *fnode; + struct buffer_head *bh; + *small = *big = 0; + + if (unlikely(is_deleted_file(f))) + return; + + fnode_block = spadfs_read_fnode_block(fs, f->fnode_block, &bh, + SRFB_FNODE | SRFB_FIXED_FNODE, + "spadfs_get_dir_hint"); + + if (unlikely(IS_ERR(fnode_block))) + return; + + fnode = (struct fnode *)((char *)fnode_block + f->fnode_pos); + *small = SPAD2CPU16_LV(&fnode->run1n); + *big = SPAD2CPU16_LV(&fnode->run2n); + spadfs_brelse(fs, bh); +} + +/* Update extended attributes */ + +static void write_ea(SPADFNODE *f, struct fnode *fnode) +{ + unsigned long ea = (unsigned long)fnode + + FNODE_EA_POS(*(volatile u8 *)&fnode->namelen); + if (unlikely(((ea + f->ea_size - 1) & + ~(unsigned long)(FNODE_BLOCK_SIZE - 1)) != + ((unsigned long)fnode & + ~(unsigned long)(FNODE_BLOCK_SIZE - 1)))) { + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "can't write extended attributes on fnode %Lx/%x(%lx)", + (unsigned long long)f->fnode_block, + f->fnode_pos, + (unsigned long)fnode & (FNODE_BLOCK_SIZE - 1)); + return; + } + if (unlikely((SPAD2CPU16_LV(&fnode->next) & FNODE_NEXT_SIZE) != + FNODE_SIZE(fnode->namelen, f->ea_size))) { + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "can't write extended attributes on fnode %Lx/%x(%lx): fnode_size %x, namelen %x, ea_size %x", + (unsigned long long)f->fnode_block, + f->fnode_pos, + (unsigned long)fnode & (FNODE_BLOCK_SIZE - 1), + (unsigned)SPAD2CPU16_LV(&fnode->next), + fnode->namelen, + f->ea_size); + return; + } + memcpy((void *)ea, f->ea, f->ea_size); +} + +static void set_spadfs_file(SPADFNODE *f, struct fnode *fnode, int part) +{ + loff_t is, pa; + loff_t disk_othersize, want_othersize; + + fnode->ctime = CPU2SPAD32(inode_get_ctime(inode(f)).tv_sec); + fnode->mtime = CPU2SPAD32(inode(f)->i_mtime.tv_sec); + fnode->flags = 0; + if (unlikely(S_ISLNK(inode(f)->i_mode))) { + u64 ino; + gid_t gid; + ino = f->stable_ino; + fnode->run10 = CPU2SPAD32(ino); + fnode->run11 = CPU2SPAD16(ino >> 32); + fnode->run1n = CPU2SPAD16(ino >> 48); + fnode->run20 = CPU2SPAD32(i_uid_read(inode(f))); + gid = i_gid_read(inode(f)); + fnode->run21 = CPU2SPAD16(gid); + fnode->run2n = CPU2SPAD16(gid >> 16); + fnode->anode0 = MAKE_PART_0(0); + fnode->anode1 = MAKE_PART_1(0); + CPU2SPAD64_LV(&fnode->size[0], 0); + CPU2SPAD64_LV(&fnode->size[1], 0); + return; + } + + fnode->run10 = MAKE_PART_0(f->blk1); + fnode->run11 = MAKE_PART_1(f->blk1); + fnode->run1n = CPU2SPAD16(f->blk1_n); + fnode->run20 = MAKE_PART_0(f->blk2); + fnode->run21 = MAKE_PART_1(f->blk2); + fnode->run2n = CPU2SPAD16(f->blk2_n); + fnode->anode0 = MAKE_PART_0(f->root); + fnode->anode1 = MAKE_PART_1(f->root); + + is = i_size_read(inode(f)); + + if (unlikely(is > f->disk_size)) { + CPU2SPAD64_LV(&fnode->size[part], f->disk_size); + pa = 0; + } else if (likely(is + (512U << f->fs->sectors_per_disk_block_bits) > f->disk_size)) { + CPU2SPAD64_LV(&fnode->size[part], is); + pa = 0; + } else { + pa = f->disk_size - (512U << f->fs->sectors_per_disk_block_bits) + 1; + CPU2SPAD64_LV(&fnode->size[part], pa); + pa -= is; + if (unlikely((pa & ~0xffffffffULL) != 0)) + pa = 0xffffffff; + } + + if (likely(f->ea_unx != NULL)) + CPU2SPAD32_LV(&f->ea_unx->prealloc[part], pa); + + disk_othersize = spadfs_roundup_blocksize(f->fs, + SPAD2CPU64_LV(&fnode->size[part ^ 1])); + if (likely(f->commit_sequence == f->fs->commit_sequence)) + want_othersize = f->crash_disk_size; + else + want_othersize = f->disk_size; + if (unlikely(disk_othersize != want_othersize)) { + CPU2SPAD64_LV(&fnode->size[part ^ 1], want_othersize); + if (likely(f->ea_unx != NULL)) + CPU2SPAD32_LV(&f->ea_unx->prealloc[part ^ 1], 0); + } +} + +int spadfs_write_file(SPADFNODE *f, int datasync, int *optimized, + struct buffer_head **bhp) +{ + struct fnode_block *fnode_block; + struct fnode *fnode; + struct buffer_head *bh; + unsigned old_prealloc; + SPADFS *fs = f->fs; + + BUG_ON(S_ISDIR(inode(f)->i_mode)); + + if (unlikely(is_deleted_file(f))) + return 0; + + fnode_block = spadfs_read_fnode_block(fs, f->fnode_block, &bh, + SRFB_FNODE | SRFB_FIXED_FNODE, + "spadfs_write_file"); + if (unlikely(IS_ERR(fnode_block))) + return PTR_ERR(fnode_block); + + fnode = (struct fnode *)((char *)fnode_block + f->fnode_pos); + + start_concurrent_atomic_buffer_modify(fs, bh); + if (unlikely((SPAD2CPU16_LV(&fnode->next) & FNODE_NEXT_SIZE) != + FNODE_SIZE(fnode->namelen, f->ea_size))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "read different file fnode at %Lx/%x: %x", + (unsigned long long)f->fnode_block, + f->fnode_pos, + SPAD2CPU16_LV(&fnode->next)); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + return -EFSERROR; + } + + old_prealloc = 0; /* against warning */ +#ifdef SPADFS_OPTIMIZE_FSYNC + if (!CC_CURRENT(fs, &fnode->cc, &fnode->txc)) { + if (spadfs_roundup_blocksize(fs, SPAD2CPU64_LV( + &fnode->size[!CC_VALID(fs, &fnode->cc, &fnode->txc)])) == + f->disk_size) { + if (unlikely(optimized != NULL)) { + if (datasync && f->ea_unx) + old_prealloc = SPAD2CPU32_LV( + &f->ea_unx->prealloc[!CC_VALID(fs, + &fnode->cc, &fnode->txc)]); + *optimized = 1; + } + goto optimize_it; + } + CC_SET_CURRENT(fs, &fnode->cc, &fnode->txc); + CPU2SPAD16_LV(&fnode->next, SPAD2CPU16_LV(&fnode->next) & + ~FNODE_NEXT_FREE); + } +optimize_it: +#endif + set_spadfs_file(f, fnode, !CC_VALID(fs, &fnode->cc, &fnode->txc)); + write_ea(f, fnode); + do_fnode_block_checksum(fs, fnode_block); + +#ifdef SPADFS_OPTIMIZE_FSYNC + if (unlikely(datasync) && likely(optimized != NULL) && + likely(*optimized) && likely(!buffer_dirty(bh))) { + unsigned current_prealloc = SPAD2CPU32_LV(&f->ea_unx->prealloc[ + CC_VALID(fs, &fnode->cc, &fnode->txc) ^ + CC_CURRENT(fs, &fnode->cc, &fnode->txc) ^ 1]); + if (old_prealloc == current_prealloc) { + end_concurrent_atomic_buffer_modify_nodirty(fs, bh); + goto ret_0; + } + } +#endif + + end_concurrent_atomic_buffer_modify(fs, bh); + +#ifdef SPADFS_OPTIMIZE_FSYNC +ret_0: +#endif + if (likely(!bhp)) + spadfs_brelse(fs, bh); + else + *bhp = bh; + + return 0; +} + +static void set_spadfs_directory(SPADFNODE *f, struct fnode *fnode, int part) +{ + fnode->ctime = CPU2SPAD32(inode_get_ctime(inode(f)).tv_sec); + fnode->mtime = CPU2SPAD32(inode(f)->i_mtime.tv_sec); + fnode->flags = FNODE_FLAGS_DIR; + CPU2SPAD64_LV(&fnode->size[part], i_size_read(inode(f))); + if (!part) { + fnode->run10 = MAKE_PART_0(f->root); + fnode->run11 = MAKE_PART_1(f->root); + } else { + fnode->run20 = MAKE_PART_0(f->root); + fnode->run21 = MAKE_PART_1(f->root); + } +} + +int spadfs_write_directory(SPADFNODE *f) +{ + struct fnode_block *fnode_block; + struct fnode *fnode; + struct buffer_head *bh; + SPADFS *fs = f->fs; + + BUG_ON(!S_ISDIR(inode(f)->i_mode)); + + if (unlikely(is_deleted_file(f))) + return 0; + + fnode_block = spadfs_read_fnode_block(fs, f->fnode_block, &bh, + SRFB_FNODE | SRFB_FIXED_FNODE, + "spadfs_write_directory"); + if (unlikely(IS_ERR(fnode_block))) + return PTR_ERR(fnode_block); + + fnode = (struct fnode *)((char *)fnode_block + f->fnode_pos); + + start_concurrent_atomic_buffer_modify(fs, bh); + if (unlikely((SPAD2CPU16_LV(&fnode->next) & FNODE_NEXT_SIZE) != + FNODE_SIZE(fnode->namelen, f->ea_size))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "read different directory fnode at %Lx/%x: %x", + (unsigned long long)f->fnode_block, + f->fnode_pos, + SPAD2CPU16_LV(&fnode->next)); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + return -EFSERROR; + } + if (!CC_CURRENT(fs, &fnode->cc, &fnode->txc)) { + CC_SET_CURRENT(fs, &fnode->cc, &fnode->txc); + CPU2SPAD16_LV(&fnode->next, SPAD2CPU16_LV(&fnode->next) & + ~FNODE_NEXT_FREE); + } + set_spadfs_directory(f, fnode, !CC_VALID(fs, &fnode->cc, &fnode->txc)); + write_ea(f, fnode); + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + return 0; +} + +static int spadfs_find_leaf_page_for_hash(SPADFNODE *fn, sector_t root, + hash_t hash, sector_t *fnode_sector, + sector_t *prev_dnode, int *hash_bits, + int *splitable) +{ + SPADFS *fs = fn->fs; + struct buffer_head *bh; + struct fnode_block *f; + *prev_dnode = 0; + *hash_bits = 0; + *splitable = -1; +subnode: + f = spadfs_read_fnode_block(fs, root, &bh, SRFB_FNODE | SRFB_DNODE, + "spadfs_find_leaf_page_for_hash"); + *fnode_sector = root; + if (unlikely(IS_ERR(f))) + return PTR_ERR(f); + + if (likely(SPAD2CPU32_LV(&f->magic) == + spadfs_magic(fs, root, FNODE_BLOCK_MAGIC))) { + spadfs_brelse(fs, bh); + return 0; + } else { + int i, j; + int r; + sector_t new_root; + hash_t hpos; + int version = dnode_version(fs, (struct dnode_page *)f); + spadfs_brelse(fs, bh); + + *prev_dnode = root; + hpos = hash & ((1 << (fs->dnode_hash_bits)) - 1); + + r = spadfs_read_dnode(fs, root, version, hpos, &new_root); + if (unlikely(r)) + return r; + + if (unlikely(!new_root)) { + sector_t new_page_sector; + int x; + int step; + r = spadfs_alloc_leaf_page(fn, root, 1U << fs->sectors_per_fnodepage_bits, 0, &new_page_sector, 0); + if (unlikely(r == -ENOSPC) || unlikely(r == -EDQUOT)) + r = spadfs_alloc_leaf_page(fn, root, 1U << fs->sectors_per_disk_block_bits, 0, &new_page_sector, 0); + if (unlikely(r)) + return r; + version = spadfs_begin_modify_dnode(fs, root); + if (unlikely(version < 0)) + return version; + r = spadfs_write_dnode(fs, root, version, hpos, new_page_sector); + if (unlikely(r)) + return r; + step = 1 << fs->dnode_hash_bits; + do { + hpos ^= step >> 1; + for (x = hpos; x < 1 << fs->dnode_hash_bits; x += step) { + sector_t test; + r = spadfs_read_dnode(fs, root, version, + x, &test); + if (unlikely(r)) + return r; + if (test) + goto done; + } + for (x = hpos; x < 1 << fs->dnode_hash_bits; x += step) { + r = spadfs_write_dnode(fs, root, + version, x, + new_page_sector); + if (unlikely(r)) + return r; + } + step >>= 1; + hpos &= ~step; + } while (step > 1); +done: + root = new_page_sector; + goto subnode; + } + if (unlikely(*hash_bits % (unsigned)fs->dnode_hash_bits) || + unlikely(*splitable != -1)) + goto bad_tree; + + j = 1 << fs->dnode_hash_bits; + for (i = 1; i < j; i <<= 1) { + sector_t test = 0; /* against warning */ + r = spadfs_read_dnode(fs, root, version, hpos ^ i, + &test); + if (unlikely(r)) + return r; + if (test == new_root) { + *splitable = hpos; + break; + } + (*hash_bits)++; + } + hash >>= fs->dnode_hash_bits; + if (unlikely(*hash_bits > SPADFS_HASH_BITS)) + goto bad_tree; + root = new_root; + goto subnode; + } +bad_tree: + spadfs_error(fs, TXFLAGS_FS_ERROR, + "bad dnode tree on %Lx (parent %Lx/%x)", + (unsigned long long)root, + (unsigned long long)fn->fnode_block, + fn->fnode_pos); + return -EFSERROR; +} + +#define HASH_1_SHIFT 16 +#define HASH_1 (1 << HASH_1_SHIFT) +#define HASH_VAL (HASH_1 - 1) + +static int test_hash_bit(char *name, unsigned namelen, unsigned hash_bits) +{ + hash_t hash = name_len_hash(name, namelen); + return ((hash >> (hash_bits & HASH_VAL)) ^ + (hash_bits >> HASH_1_SHIFT) ^ 1) & 1; +} + +static int test_hash(SPADFS *fs, sector_t sec, int hash_bits) +{ + hash_t hash; + int found = 0; + struct buffer_head *bh; + struct fnode_block *fnode_block; + struct fnode *fnode; + unsigned size, namelen; +next_block: + fnode_block = spadfs_read_fnode_block(fs, sec, &bh, SRFB_FNODE, + "test_hash"); + if (unlikely(IS_ERR(fnode_block))) + return PTR_ERR(fnode_block); + + fnode = fnode_block->fnodes; + +next_fnode: + VALIDATE_FNODE(fs, fnode_block, fnode, size, namelen, + used, free, bad_fnode); + +used: + hash = name_len_hash(FNODE_NAME(fnode), namelen); + found |= 1 << ((hash >> hash_bits) & 1); + if (found == 3) { + spadfs_brelse(fs, bh); + return 3; + } + +free: + fnode = (struct fnode *)((char *)fnode + size); + if (likely(((unsigned long)fnode & (FNODE_BLOCK_SIZE - 1)) != 0)) + goto next_fnode; + + if (!(fnode_block->flags & FNODE_BLOCK_LAST)) { + spadfs_brelse(fs, bh); + sec++; + goto next_block; + } + spadfs_brelse(fs, bh); + return found; + +bad_fnode: + spadfs_brelse(fs, bh); + spadfs_error(fs, TXFLAGS_FS_ERROR, + "test_hash: bad fnode on block %Lx", + (unsigned long long)sec); + return -EFSERROR; +} + +/* + * Copy fnodes from fnode blocks starting with src_sec up to the block with + * FNODE_BLOCK_LAST to fnode blocks starting with dst_sec up to the block with + * FNODE_BLOCK_LAST. Destination must be largrer or equal than source. Fix + * pointer on in-memory fnoes. + * Copy only fnodes that match condition (x is a cookie passed to it). + * If preserve_pos is true, we must preserve positions of entries, otherwise + * readdir could skip them. + */ + +static int copy_fnodes_to_block(SPADFS *fs, sector_t dst_sec, sector_t src_sec, + int (*condition)(char *name, unsigned namelen, + unsigned x), + unsigned x, int preserve_pos) +{ + struct fnode_block *src_block, *dst_block; + struct fnode *src_fnode, *dst_fnode; + struct buffer_head *bh_src, *bh_dst; + unsigned size, dsize; + unsigned namelen; + + dst_block = spadfs_read_fnode_block(fs, dst_sec, &bh_dst, SRFB_FNODE, + "copy_fnodes_to_block 1"); + if (unlikely(IS_ERR(dst_block))) + return PTR_ERR(dst_block); + + dst_fnode = dst_block->fnodes; + dsize = FNODE_MAX_SIZE; + +new_src_block: + src_block = spadfs_read_fnode_block(fs, src_sec, &bh_src, SRFB_FNODE, + "copy_fnodes_to_block 2"); + if (unlikely(IS_ERR(src_block))) { + spadfs_brelse(fs, bh_dst); + return PTR_ERR(src_block); + } + src_fnode = src_block->fnodes; + +next_fnode: + VALIDATE_FNODE(fs, src_block, src_fnode, size, namelen, + ok, skip, bad_fnode); + +ok: + if (condition && !condition(FNODE_NAME(src_fnode), namelen, x)) { +skip: + if (preserve_pos) { + /* + * adding new entry will join successive free fnodes anyway, no + * need to mess with it now + */ + if (unlikely(dsize < size)) { + if (unlikely(dsize)) + spadfs_error(fs, TXFLAGS_FS_ERROR, + "destination is not in sync with source, from %Lx to %Lx, dsize %u, size %u", + (unsigned long long)src_sec, + (unsigned long long)dst_sec, + dsize, size); + goto new_dest; + } + CPU2SPAD16_LV(&dst_fnode->next, size | FNODE_NEXT_FREE); + CPU2SPAD16_LV(&dst_fnode->cc, 0); + CPU2SPAD32_LV(&dst_fnode->txc, 0); + goto new_dst_fnode; + } + goto new_src_fnode; + } + if (likely(dsize >= size)) { + memcpy(dst_fnode, src_fnode, size); + spadfs_move_fnode_ptr(fs, src_sec, + (char *)src_fnode - (char *)src_block, + dst_sec, + (char *)dst_fnode - (char *)dst_block, + src_fnode->flags & FNODE_FLAGS_DIR); +new_dst_fnode: + if (likely(dsize -= size)) { + dst_fnode = (struct fnode *)((char *)dst_fnode + size); + CPU2SPAD16_LV(&dst_fnode->next, + dsize | FNODE_NEXT_FREE); + CPU2SPAD16_LV(&dst_fnode->cc, 0); + CPU2SPAD32_LV(&dst_fnode->txc, 0); + } +new_src_fnode: + src_fnode = (struct fnode *)((char *)src_fnode + size); + if (unlikely(!((unsigned long)src_fnode & + (FNODE_BLOCK_SIZE - 1)))) { + if (unlikely((src_block->flags & FNODE_BLOCK_LAST) != + 0)) + goto end; + spadfs_brelse(fs, bh_src); + src_sec++; + goto new_src_block; + } + goto next_fnode; + } else { +new_dest: + do_fnode_block_checksum(fs, dst_block); + mark_buffer_dirty(bh_dst); + if (unlikely(dst_block->flags & FNODE_BLOCK_LAST)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "ran over destination when copying fnode blocks from %Lx to %Lx", + (unsigned long long)src_sec, + (unsigned long long)dst_sec); + spadfs_brelse(fs, bh_dst); + spadfs_brelse(fs, bh_src); + return -EFSERROR; + } + spadfs_brelse(fs, bh_dst); + dst_sec++; + dst_block = spadfs_read_fnode_block(fs, dst_sec, &bh_dst, + SRFB_FNODE, + "copy_fnodes_to_block 3"); + if (unlikely(IS_ERR(dst_block))) { + spadfs_brelse(fs, bh_src); + return PTR_ERR(dst_block); + } + dst_fnode = dst_block->fnodes; + dsize = FNODE_MAX_SIZE; + goto next_fnode; + } +end: + spadfs_brelse(fs, bh_src); + do_fnode_block_checksum(fs, dst_block); + mark_buffer_dirty(bh_dst); + spadfs_brelse(fs, bh_dst); + return 0; + +bad_fnode: + spadfs_brelse(fs, bh_src); + spadfs_brelse(fs, bh_dst); + spadfs_error(fs, TXFLAGS_FS_ERROR, + "bad fnode on block %Lx when copying fnodes", + (unsigned long long)src_sec); + return -EFSERROR; +} + +struct fnode *spadfs_add_fnode_to_directory(SPADFNODE *dir, + const char *name, unsigned namelen, + unsigned ea_size, + struct buffer_head **bhp, + sector_t *fnode_address, + unsigned *fnode_off, + struct fnode_block **pfnode_block, + int wlock) +{ + SPADFS *fs = dir->fs; + sector_t c[2]; + int hash_bits, splitable; + int r; + int pass2; + unsigned long long chains; + sector_t fnode_blk_sector; + unsigned n_sec; + sector_t dnode; + int size, rsize; + unsigned xnamelen; + int restarts; + struct fnode *fnode; + struct fnode_block *fnode_block; + hash_t hash; + + c[1] = 0; + + if (unlikely(namelen > MAX_NAME_LEN)) + return ERR_PTR(-ENAMETOOLONG); + hash = name_len_hash(name, namelen); + BUG_ON(ea_size > FNODE_MAX_EA_SIZE); + rsize = FNODE_SIZE(namelen, ea_size); + restarts = 0; + +total_restart: + if (unlikely(++restarts > SPADFS_HASH_BITS + 1)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "too many splits at directory %Lx --- probably unreliable device", + (unsigned long long)dir->root); + return ERR_PTR(-EFSERROR); + } + + r = spadfs_find_leaf_page_for_hash(dir, dir->root, hash, + &fnode_blk_sector, &dnode, + &hash_bits, &splitable); + if (unlikely(r)) + return ERR_PTR(r); + + pass2 = 0; + chains = 0; + +next_fnode_page: + n_sec = 0; + +next_fnode_block: + + fnode_block = spadfs_read_fnode_block(fs, fnode_blk_sector + n_sec, bhp, + SRFB_FNODE, + "spadfs_add_fnode_to_directory"); + if (unlikely(IS_ERR(fnode_block))) + return (void *)fnode_block; + + fnode = fnode_block->fnodes; + +next_fnode: +retry_fnode: + VALIDATE_FNODE(fs, fnode_block, fnode, size, xnamelen, + used, free, bad_fnode); + +free: + if (!CC_CURRENT(fs, &fnode->cc, &fnode->txc)) { + if (size >= rsize) { + *fnode_address = fnode_blk_sector + n_sec; + *fnode_off = (char *)fnode - (char *)fnode_block; + *pfnode_block = fnode_block; + start_concurrent_atomic_buffer_modify(fs, *bhp); + if (rsize < size) { + struct fnode *afterfnode = (struct fnode *) + ((char *)fnode + rsize); + CPU2SPAD16_LV(&afterfnode->next, + (size - rsize) | FNODE_NEXT_FREE); + CPU2SPAD16_LV(&afterfnode->cc, 0); + CPU2SPAD32_LV(&afterfnode->txc, 0); + CPU2SPAD16_LV(&fnode->next, + rsize | FNODE_NEXT_FREE); + } + CC_SET_CURRENT_INVALID(fs, &fnode->cc, &fnode->txc); + return fnode; + } else { + struct fnode *new_fnode = (struct fnode *) + ((char *)fnode + size); + int new_size; + if (unlikely(!((unsigned long) new_fnode & + (FNODE_BLOCK_SIZE - 1)))) + goto end_of_block; + new_size = SPAD2CPU16_LV(&new_fnode->next) & FNODE_NEXT_SIZE; + VALIDATE_FNODE(fs, fnode_block, new_fnode, new_size, + xnamelen, + s_used, s_free, bad_fnode); +s_free: + if (likely(!CC_CURRENT(fs, &new_fnode->cc, + &new_fnode->txc))) { + start_concurrent_atomic_buffer_modify(fs, *bhp); + CPU2SPAD16_LV(&fnode->next, + SPAD2CPU16_LV(&fnode->next) + new_size); + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, *bhp); + goto retry_fnode; + } +s_used: + fnode = new_fnode; + size = new_size; + } + } + +used: + fnode = (struct fnode *)((char *)fnode + size); + if (likely(((unsigned long)fnode & (FNODE_BLOCK_SIZE - 1)) != 0)) + goto next_fnode; + +end_of_block: + if (!(fnode_block->flags & FNODE_BLOCK_LAST)) { + n_sec++; + spadfs_brelse(fs, *bhp); + goto next_fnode_block; + } + + if (CC_VALID(fs, &fnode_block->cc, &fnode_block->txc)) { + fnode_blk_sector = MAKE_D_OFF(fnode_block->next0, + fnode_block->next1); + spadfs_brelse(fs, *bhp); + if (unlikely(spadfs_stop_cycles(fs, fnode_blk_sector, &c, + "spadfs_add_fnode_to_directory"))) + return ERR_PTR(-EFSERROR); + pass2 = 1; + chains++; + goto next_fnode_page; + } + if (CC_CURRENT(fs, &fnode_block->cc, &fnode_block->txc)) { + spadfs_brelse(fs, *bhp); + return NEED_SYNC; + } + if (likely(!pass2)) { + int th; + sector_t fnode_block_0_sector; + sector_t fnode_block_1_sector; + int version; + int bits; + + if (!wlock) { + spadfs_brelse(fs, *bhp); + return NEED_WLOCK; + } + +/* 1. REALLOC CURRENT BLOCK IF IT'S ROOT & SMALL */ + + /* + * warning, n_sec is the last sector number + * --- i.e. there are n_sec+1 sectors + */ + if (!dnode && (n_sec + 1) * 2 < + 1U << fs->sectors_per_fnodepage_bits) { + unsigned new_sectors; + sector_t new_sector; + new_sectors = FNODE_BLOCK_SIZE / 512; + while (new_sectors < (n_sec + 1) * 2) + new_sectors <<= 1; + r = spadfs_alloc_leaf_page(dir, fnode_blk_sector, + new_sectors, 0, + &new_sector, 0); + if (unlikely(r)) { + if (likely(r == -ENOSPC) || r == -EDQUOT) + goto alloc_chain; + goto brelse_return_r; + } + spadfs_brelse(fs, *bhp); + /* + * trick: start copying to new_sector + n_sec + 1 (i.e. + * entirely newly allocated blocks) to make sure that + * readdir won't skip any entry. Otherwise we'd have to + * set preserve_pos --- it would waste space more. + */ + r = copy_fnodes_to_block(fs, new_sector + n_sec + 1, + fnode_blk_sector, NULL, 0, 0); + if (unlikely(r)) { + spadfs_free_directory_blocks(dir, new_sector, + new_sectors); + return ERR_PTR(r); + } + spadfs_free_directory_blocks(dir, fnode_blk_sector, + n_sec + 1); + /* + * don't handle spadfs_free_directory_blocks error + * --- we can't revert copied fnodes + */ + dir->root = new_sector; + r = spadfs_write_directory(dir); + if (unlikely(r)) + return ERR_PTR(r); + fnode_blk_sector = new_sector; + goto next_fnode_page; + } + +/* 2. IF IT'S END OF HASH, ALLOC CHAIN */ + + if (unlikely(hash_bits >= SPADFS_HASH_BITS)) + goto alloc_chain; + +/* 3. IF THE DNODE IS FULL, ALLOC NEW DNODE */ + + if (splitable == -1) { + sector_t new_dnode; + r = spadfs_alloc_dnode_page(dir, fnode_blk_sector, + &new_dnode, dnode, + fnode_blk_sector); + if (unlikely(r)) { + if (likely(r == -ENOSPC) || r == -EDQUOT) + goto alloc_chain; + goto brelse_return_r; + } + spadfs_brelse(fs, *bhp); + if (dnode) { + unsigned i; + int version = spadfs_begin_modify_dnode(fs, + dnode); + if (unlikely(version < 0)) + return ERR_PTR(version); + for (i = 0; i < 1 << fs->dnode_hash_bits; i++) { + sector_t test; + r = spadfs_read_dnode(fs, dnode, + version, i, &test); + if (unlikely(r)) + return ERR_PTR(r); + if (test == fnode_blk_sector) + spadfs_write_dnode(fs, dnode, + version, i, + new_dnode); + } + } else { + dir->root = new_dnode; + r = spadfs_write_directory(dir); + if (unlikely(r)) + return ERR_PTR(r); + } + dnode = new_dnode; + splitable = 0; + goto next_fnode_page; + } + +/* 4. TEST SPLITABILITY OF FNODE PAGE */ + + th = test_hash(fs, fnode_blk_sector, hash_bits); + if (unlikely(th < 0)) { + spadfs_brelse(fs, *bhp); + return ERR_PTR(th); + } + +/* 5. ALLOC NEW 2 FNODE PAGES AND SPLIT FNODE */ + + if (th & 1) { + r = spadfs_alloc_leaf_page(dir, fnode_blk_sector, + 1U << fs->sectors_per_fnodepage_bits, 0, + &fnode_block_0_sector, 0); + if (unlikely(r)) { + if (likely(r == -ENOSPC) || r == -EDQUOT) + goto alloc_chain; + goto brelse_return_r; + } + } else + fnode_block_0_sector = 0; + + if (th & 2) { + r = spadfs_alloc_leaf_page(dir, fnode_blk_sector, + 1U << fs->sectors_per_fnodepage_bits, 0, + &fnode_block_1_sector, 0); + if (unlikely(r)) { + if (fnode_block_0_sector) { + int rr; + rr = spadfs_free_directory_blocks( + dir, + fnode_block_0_sector, + 1U << fs->sectors_per_fnodepage_bits); + if (unlikely(rr)) { + r = rr; + goto brelse_return_r; + } + } + if (likely(r == -ENOSPC) || r == -EDQUOT) + goto alloc_chain; + goto brelse_return_r; + } + + r = copy_fnodes_to_block(fs, + fnode_block_1_sector, fnode_blk_sector, + test_hash_bit, hash_bits | HASH_1, 0); + if (unlikely(r)) + goto brelse_return_r; + } else + fnode_block_1_sector = 0; + if (th & 1) { + r = copy_fnodes_to_block(fs, + fnode_block_0_sector, fnode_blk_sector, + test_hash_bit, hash_bits, 1); + if (unlikely(r)) + goto brelse_return_r; + } + + spadfs_brelse(fs, *bhp); + +/* 6. READ PARENT DNODE */ + + version = spadfs_begin_modify_dnode(fs, dnode); + if (unlikely(version < 0)) + return ERR_PTR(version); + +/* 7. SPLIT POINTERS ON DNODE */ + + bits = hash_bits % (unsigned)fs->dnode_hash_bits; + splitable &= (1 << bits) - 1; + do { + r = spadfs_write_dnode(fs, dnode, version, splitable, + fnode_block_0_sector); + if (unlikely(r)) + return ERR_PTR(r); + splitable += 1 << bits; + r = spadfs_write_dnode(fs, dnode, version, splitable, + fnode_block_1_sector); + if (unlikely(r)) + return ERR_PTR(r); + splitable += 1 << bits; + } while (splitable < 1 << fs->dnode_hash_bits); + r = spadfs_free_directory_blocks(dir, fnode_blk_sector, + n_sec + 1); + if (unlikely(r)) + return ERR_PTR(r); + goto total_restart; + } else { + if (unlikely(pass2 == 2)) { + spadfs_brelse(fs, *bhp); + spadfs_error(fs, TXFLAGS_FS_ERROR, + "couldn't allocate fnode even in new page"); + return ERR_PTR(-EFSERROR); + } + +alloc_chain: + if (unlikely(chains >= MAX_CHAIN_LENGTH)) { + /* we couldn't readdir it on Linux. */ + spadfs_brelse(fs, *bhp); + return ERR_PTR(-ENOSPC); + } + r = spadfs_alloc_leaf_page(dir, fnode_blk_sector + n_sec, + 1U << fs->sectors_per_disk_block_bits, + fnode_blk_sector + n_sec, + &fnode_blk_sector, 0); + if (unlikely(r)) + goto brelse_return_r; + + start_concurrent_atomic_buffer_modify(fs, *bhp); + fnode_block->next0 = MAKE_PART_0(fnode_blk_sector); + fnode_block->next1 = MAKE_PART_1(fnode_blk_sector); + CPU2SPAD16_LV(&fnode_block->cc, fs->cc); + CPU2SPAD32_LV(&fnode_block->txc, fs->txc); + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, *bhp); + spadfs_brelse(fs, *bhp); + + pass2 = 2; + chains++; + goto next_fnode_page; + } + +brelse_return_r: + spadfs_brelse(fs, *bhp); + return ERR_PTR(r); + +bad_fnode: + spadfs_brelse(fs, *bhp); + spadfs_error(fs, TXFLAGS_FS_ERROR, + "bad fnode on block %Lx when adding to directory", + (unsigned long long)(fnode_blk_sector + n_sec)); + return ERR_PTR(-EFSERROR); +} + +static noinline void spadfs_swap_fnode(SPADFNODE *file, struct fnode *fnode) +{ + u64 s; + + unsigned fnode_size, ea_offset; + struct ea_unx *ea_unx; + + if (unlikely(fnode->flags & FNODE_FLAGS_DIR)) { + u32 r0 = fnode->run10; + u16 r1 = fnode->run11; + fnode->run10 = fnode->run20; + fnode->run11 = fnode->run21; + fnode->run20 = r0; + fnode->run21 = r1; + } + s = fnode->size[0]; + fnode->size[0] = fnode->size[1]; + fnode->size[1] = s; + + /* + Warning: this can't be used here, in case we are growing ea and + refiling, we'd smash fnode block here. + if (file->ea_unx) { + u32 pa = file->ea_unx->prealloc[0]; + file->ea_unx->prealloc[0] = file->ea_unx->prealloc[1]; + file->ea_unx->prealloc[1] = pa; + write_ea(file, fnode); + } + */ + + ea_offset = FNODE_EA_POS(fnode->namelen); + fnode_size = SPAD2CPU16_LV(&fnode->next) & FNODE_NEXT_SIZE; + if (unlikely(ea_offset > fnode_size) || + unlikely(((unsigned long)fnode & (FNODE_BLOCK_SIZE - 1)) + + fnode_size > FNODE_BLOCK_SIZE)) { + spadfs_error(file->fs, TXFLAGS_FS_ERROR, + "spadfs_swap_fnode: invalid fnode"); + return; + } + ea_unx = (struct ea_unx *)GET_EA((void *)((u8 *)fnode + ea_offset), + fnode_size - ea_offset, + EA_UNX_MAGIC, EA_UNX_MAGIC_MASK); + if (unlikely(ea_unx == GET_EA_ERROR)) { + spadfs_error(file->fs, TXFLAGS_FS_ERROR, + "spadfs_swap_fnode: invalid extended attributes"); + return; + } + if (likely(ea_unx != NULL)) { + u32 pa = ea_unx->prealloc[0]; + ea_unx->prealloc[0] = ea_unx->prealloc[1]; + ea_unx->prealloc[1] = pa; + } +} + +int spadfs_remove_fnode_from_directory(SPADFNODE *dir, SPADFNODE *file, + struct qstr *name) +{ + SPADFS *fs = dir->fs; + struct fnode_block *fnode_block; + struct fnode *fnode; + struct buffer_head *bh; + spadfs_ino_t ino; + int r; + + if (likely(file != NULL)) { + ino = file->spadfs_ino; + if (unlikely(is_fnode_fixed(file))) { + file = NULL; + r = spadfs_lookup_ino(dir, name, &ino, 1); + if (unlikely(r)) + goto lookup_error; + } + } else { + /* + * This branch is taken if we encounter an error when creating + * a new file and we need to delete the directory entry. + */ + ino = 0; /* against warning */ + r = spadfs_lookup_ino(dir, name, &ino, 0); + if (unlikely(r)) + goto lookup_error; + } + + fnode_block = spadfs_read_fnode_block(fs, spadfs_ino_t_sec(ino), &bh, + SRFB_FNODE, + "spadfs_remove_fnode_from_directory"); + if (unlikely(IS_ERR(fnode_block))) + return PTR_ERR(fnode_block); + + fnode = (struct fnode *)((char *)fnode_block + spadfs_ino_t_pos(ino)); + start_concurrent_atomic_buffer_modify(fs, bh); + if (CC_CURRENT(fs, &fnode->cc, &fnode->txc) && + SPAD2CPU16_LV(&fnode->next) & FNODE_NEXT_FREE) { + CPU2SPAD16_LV(&fnode->cc, 0); + CPU2SPAD32_LV(&fnode->txc, 0); + } else { + if (likely(file != NULL) && + CC_VALID(fs, &fnode->cc, &fnode->txc) ^ + CC_CURRENT(fs, &fnode->cc, &fnode->txc)) + spadfs_swap_fnode(file, fnode); + CPU2SPAD16_LV(&fnode->cc, fs->cc); + CPU2SPAD32_LV(&fnode->txc, fs->txc); + CPU2SPAD16_LV(&fnode->next, SPAD2CPU16_LV(&fnode->next) | + FNODE_NEXT_FREE); + } + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + return 0; + +lookup_error: + if (r > 0) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "can't find entry to delete in directory %Lx/%x", + (unsigned long long)dir->fnode_block, + dir->fnode_pos); + r = -EFSERROR; + } + return r; +} + +/* + * Move the fnode "old_file" with name "old_name" from "old_dir" to + * the directory "new_dir" under "new_name". + * + * If there was another fnode with "new_name" in "new_dir", "new_dentry" must + * point to it --- it will be deleted. + * + * The filesystem must be locked for write. + */ +int spadfs_move_fnode_to_directory(SPADFNODE *old_dir, struct qstr *old_name, + SPADFNODE *old_file, + SPADFNODE *new_dir, struct qstr *new_name, + struct dentry *new_dentry, + u8 *new_ea, unsigned new_ea_size) +{ + SPADFS *fs = old_file->fs; + sector_t fnode_address; + unsigned fnode_off; + struct fnode_block *fnode_block; + struct fnode *fnode; + struct buffer_head *bh; + int r; + int synced = 0; + u16 hint_small, hint_big; + struct inode *new_file = new_dentry ? new_dentry->d_inode : NULL; + + assert_write_sync_lock(fs); + + /* + * If this were after spadfs_add_fnode_to_directory, it would deadlock + */ + if (unlikely(S_ISDIR(inode(old_file)->i_mode))) + spadfs_get_dir_hint(old_file, &hint_small, &hint_big); + + /* + * If moving over an existing directory, check that it's empty. + */ + if (unlikely(new_file != NULL)) { + if (unlikely(S_ISDIR(new_file->i_mode))) { + r = spadfs_check_directory_empty(spadfnode(new_file)); + if (unlikely(r)) + goto unlock_ret_r; + } + } + + /* + * Add the fnode to the new directory. + */ +again: + fnode = spadfs_add_fnode_to_directory(new_dir, + (const char *)new_name->name, new_name->len, + unlikely(is_fnode_fixed(old_file)) ? 0 : + unlikely(new_ea != NULL) ? new_ea_size : + old_file->ea_size, + &bh, &fnode_address, &fnode_off, &fnode_block, 1); + + if (unlikely(IS_ERR(fnode))) { + if (likely(fnode == ERR_PTR(-ENOSPC)) && !synced) + goto do_sync; + r = PTR_ERR(fnode); + goto unlock_ret_r; + } + if (unlikely(fnode == NEED_SYNC)) { +do_sync: + if (unlikely(r = spadfs_commit_unlocked(fs))) + goto unlock_ret_r; + synced = 1; + goto again; + } + BUG_ON(fnode == NEED_WLOCK); + + /* + * Set the fnode attributes. + */ + fnode->namelen = new_name->len; + spadfs_set_name(fs, FNODE_NAME(fnode), (const char *)new_name->name, new_name->len); + + if (unlikely(is_fnode_fixed(old_file))) { + make_fixed_fnode_reference(fnode, old_file->fnode_block); + } else { + if (likely(!S_ISDIR(inode(old_file)->i_mode))) { + set_spadfs_file(old_file, fnode, 1); + fnode->flags = 0; + } else { + set_spadfs_directory(old_file, fnode, 1); + fnode->run1n = SPAD2CPU16_LV(&hint_small); + fnode->run2n = SPAD2CPU16_LV(&hint_big); + fnode->flags = FNODE_FLAGS_DIR; + } + if (unlikely(new_ea != NULL)) { + memcpy(old_file->ea, new_ea, new_ea_size); + old_file->ea_size = new_ea_size; + spadfs_find_ea_unx(old_file); + } + write_ea(old_file, fnode); + } + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + + /* + * Remove the fnode from the old directory. + */ + spadfs_remove_fnode_from_directory(old_dir, old_file, old_name); + if (likely(!is_fnode_fixed(old_file))) { + spadfs_move_fnode_ptr(fs, + old_file->fnode_block, old_file->fnode_pos, + fnode_address, fnode_off, + S_ISDIR(inode(old_file)->i_mode)); + spadfs_set_parent_fnode(old_file, new_dir->fnode_block, new_dir->fnode_pos); + if (unlikely(old_file->fnode_block != fnode_address) || + unlikely(old_file->fnode_pos != fnode_off) || + unlikely(old_file->spadfs_ino != + make_spadfs_ino_t(fnode_address, fnode_off))) { + panic("spadfs: spadfs_move_fnode_ptr didn't do its job: %Lx != %Lx || %x != %x || %Lx != %Lx", + (unsigned long long)old_file->fnode_block, + (unsigned long long)fnode_address, + old_file->fnode_pos, fnode_off, + (unsigned long long)old_file->spadfs_ino, + (unsigned long long)make_spadfs_ino_t(fnode_address, fnode_off)); + } + } + + /* + * If some other fnode with the same name exists in the new directory, + * delete it. + */ + if (unlikely(new_file != NULL)) { + if (unlikely(S_ISDIR(new_file->i_mode))) + spadfs_remove_recursive(spadfnode(new_file), spadfnode(new_file)->root, 0); + spadfs_unlink_unlocked(new_dir, new_dentry); + } + + r = 0; +unlock_ret_r: + return r; +} + +/* + * Need to be called after change of length of extended attributes or change + * of link count --- will reinsert fnode to directory and drop old instance. + * Must be called with read lock on the filesystem and lock on file (if + * "file" argument is a file) or with write lock on the filesystem. + */ + +int spadfs_refile_fixed_fnode(SPADFNODE *file, u8 *new_ea, unsigned new_ea_size) +{ + SPADFS *fs = file->fs; + struct fnode *fnode, *old_fnode; + unsigned old_fnode_pos; + struct buffer_head *bh; + struct fixed_fnode_block *fixed_fnode_block; + + if (unlikely(is_deleted_file(file))) + return 0; + if (unlikely(!is_fnode_fixed(file))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "fnode is not fixed: %Lx/%x", + (unsigned long long)file->fnode_block, + file->fnode_pos); + return -EFSERROR; + } + + fixed_fnode_block = (struct fixed_fnode_block *) + spadfs_read_fnode_block(fs, file->fnode_block, &bh, + SRFB_FIXED_FNODE, + "spadfs_refile_fixed_fnode"); + if (unlikely(IS_ERR(fixed_fnode_block))) + return PTR_ERR(fixed_fnode_block); + + start_concurrent_atomic_buffer_modify(fs, bh); + old_fnode = (struct fnode *) + ((char *)fixed_fnode_block + file->fnode_pos); + old_fnode_pos = file->fnode_pos; + if (!CC_CURRENT(fs, &fixed_fnode_block->cc, &fixed_fnode_block->txc)) { + CC_SET_CURRENT(fs, &fixed_fnode_block->cc, + &fixed_fnode_block->txc); + file->fnode_pos = FIXED_FNODE_BLOCK_FNODE0 + + FIXED_FNODE_BLOCK_FNODE1 - file->fnode_pos; + } + + if (unlikely(new_ea != NULL)) { + memcpy(file->ea, new_ea, new_ea_size); + file->ea_size = new_ea_size; + spadfs_find_ea_unx(file); + } + + fnode = (struct fnode *)((char *)fixed_fnode_block + file->fnode_pos); + CPU2SPAD64_LV(FIXED_FNODE_NLINK_PTR(fnode), file->spadfs_nlink); + CPU2SPAD16_LV(&fnode->next, FNODE_SIZE(0, file->ea_size)); + /* We need to set current cc/txc because of possible resurrect */ + CPU2SPAD16_LV(&fnode->cc, fs->cc); + CPU2SPAD32_LV(&fnode->txc, fs->txc); + fnode->namelen = 0; + if (likely(!S_ISDIR(inode(file)->i_mode))) { + set_spadfs_file(file, fnode, 0); + fnode->flags = 0; + } else { + set_spadfs_directory(file, fnode, 0); + fnode->flags = FNODE_FLAGS_DIR; + fnode->run1n = old_fnode->run1n; /* alloc hints */ + fnode->run2n = old_fnode->run2n; + spadfs_move_parent_dir_ptr(fs, file->fnode_block, old_fnode_pos, + file->fnode_block, file->fnode_pos); + } + write_ea(file, fnode); + do_fnode_block_checksum(fs, (struct fnode_block *)fixed_fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + return 0; +} + +int spadfs_refile_fnode(SPADFNODE *dir, struct qstr *qstr, SPADFNODE *file, + u8 *new_ea, unsigned new_ea_size) +{ + SPADFS *fs = file->fs; + int r; + + assert_write_sync_lock(fs); + + if (unlikely(r = spadfs_ea_resize(file, new_ea_size))) + return r; + + if (unlikely(is_deleted_file(file))) { + memcpy(file->ea, new_ea, new_ea_size); + file->ea_size = new_ea_size; + spadfs_find_ea_unx(file); + return 0; + } + + if (is_fnode_fixed(file)) + r = spadfs_refile_fixed_fnode(file, new_ea, new_ea_size); + else + r = spadfs_move_fnode_to_directory(dir, qstr, file, dir, qstr, + NULL, new_ea, new_ea_size); + return r; +} + diff --git a/fs/spadfs/dir.h b/fs/spadfs/dir.h new file mode 100644 index 000000000..01bd2a1bf --- /dev/null +++ b/fs/spadfs/dir.h @@ -0,0 +1,167 @@ +void really_do_fnode_block_checksum(struct fnode_block *fnode_block); + +/* + * do_fnode_block_checksum is called after update to fnode block + * --- must be within + * start_concurrent_atomic_buffer_modify/end_concurrent_atomic_buffer_modify so + * that partially modified block is not written to disk. + */ + +static inline void do_fnode_block_checksum(SPADFS *fs, + struct fnode_block *fnode_block) +{ + if (unlikely(make_checksums(fs))) + really_do_fnode_block_checksum(fnode_block); + else + fnode_block->flags &= ~FNODE_BLOCK_CHECKSUM_VALID; +} + +/* + * Test fnode validity and jump one of these labels: + * ok (fnode valid) + * skip_free (free fnode) + * bad_fnode (corrupted filesystem) + * It should test for all cases of filesystem corruption and make sure that we + * won't reference out-of-bound memory later, no matter what data are on disk. + * + * fs --- pointer to filesystem + * fnode_block --- pointer to 512-byte fnode block + * fnode --- pointer to fnode within this block + * size --- fnode->next & FNODE_NEXT_SIZE is put here + * name_len --- the name length is put here + * --- the purpose of size & name_len arguments is just to not read the same + * entry more times + */ + +/* + * volatile is used to make sure that the appropriate entry is read only once. + * Without volatile, the compiler may read the entry multiple times and it could + * cause out-of-memory accesses if the structure is simultaneously being + * modified (simultaneous modification is a symptom of filesystem corruption + * anyway, but it should not cause out-of-memory accesses). + */ + +#define VALIDATE_FNODE(fs, fnode_block, fnode, size, name_len, \ + ok, skip_free, bad_fnode) \ +do { \ + (size) = *(volatile u16 *)&(fnode)->next; \ + (size) = SPAD2CPU16(size) & FNODE_NEXT_SIZE; \ + if (unlikely((((unsigned long)(fnode) + (size) - 1) & \ + ~(unsigned long)(FNODE_BLOCK_SIZE - 1)) != \ + ((unsigned long)(fnode_block) & \ + ~(unsigned long)(FNODE_BLOCK_SIZE - 1)))) \ + goto bad_fnode; \ + \ + if ((SPAD2CPU16_LV(&(fnode)->next) & FNODE_NEXT_FREE) && \ + CC_VALID((fs), &(fnode)->cc, &(fnode)->txc)) { \ + if (unlikely((size) < FNODE_HEAD_SIZE)) \ + goto bad_fnode; \ + goto skip_free; \ + } \ + \ + if (unlikely((size) <= FNODE_NAME_POS)) \ + goto bad_fnode; \ + \ + (name_len) = *(volatile u8 *)&(fnode)->namelen; \ + if (unlikely((unsigned)((size) - FNODE_EA_POS(name_len)) > \ + FNODE_MAX_EA_SIZE)) \ + goto bad_fnode; \ + \ + /* prevent speculative fetch of other entries by the compiler */\ + barrier(); \ + goto ok; \ +} while (0) + +/* + * Create a pointer to fixed fnode instead of regular file --- for this pointer, + * most fields are unused. We zero them anyway. + */ + +static inline void make_fixed_fnode_reference(struct fnode *fnode, sector_t blk) +{ + fnode->size[0] = fnode->size[1] = CPU2SPAD64_CONST(0); + fnode->ctime = fnode->mtime = CPU2SPAD32_CONST(0); + fnode->run10 = MAKE_PART_0(0); + fnode->run11 = MAKE_PART_1(0); + fnode->run1n = CPU2SPAD16_CONST(0); + fnode->run20 = MAKE_PART_0(0); + fnode->run21 = MAKE_PART_1(0); + fnode->run2n = CPU2SPAD16_CONST(0); + fnode->anode0 = MAKE_PART_0(blk); + fnode->anode1 = MAKE_PART_1(blk); + fnode->flags = FNODE_FLAGS_HARDLINK; +} + +/* + * Lock filesystem for read or write when modifying directories (many functions + * in namei.c). The usage is this: + * Lock for read, try to modify directory. + * When split is needed, NEED_WLOCK is returned --- so we unlock for read, + * lock for write and retry. + * When sync is needed (i.e. we are out of data but something can be freed + * by sync, NEED_SYNC is returned. --- so unlock, sync, retry. + * But do not sync again on NEED_SYNC to prevent livelock --- + * return -ENOSPC if NEED_SYNC is returned the second time. + * + * The rationale for locking is that when splitting directory pages, all + * filesystem activity must stop because file updates update entries in + * directories. + * + * Maybe + * start_concurrent_atomic_buffer_modify/end_concurrent_atomic_buffer_modify + * could be used --- i.e. updater reads fnode position, breads the buffer + * (without checking magic), calls start_concurrent_atomic_buffer_modify, + * rechecks the position again and if it is the same, he is now protected from + * directory operations. If anyone runs into contention on this lock, I can + * investigate this further. + */ + +#define ND_LOCK(fs, wlock) \ +do { \ + if (likely(!(wlock))) \ + down_read_sync_lock(fs); \ + else \ + down_write_sync_lock(fs); \ +} while (0) + +#define ND_UNLOCK(fs, wlock) \ +do { \ + if (likely(!(wlock))) \ + up_read_sync_lock(fs); \ + else \ + up_write_sync_lock(fs); \ +} while (0) + +int spadfs_alloc_fixed_fnode_block(SPADFNODE *fn, sector_t hint, unsigned size, + u16 hint_small, u16 hint_big, + sector_t *result); +int spadfs_alloc_leaf_page(SPADFNODE *fn, sector_t hint, unsigned sectors, + sector_t prev, sector_t *result, int noaccount); +struct fnode_block *spadfs_find_hash_block(SPADFNODE *f, hash_t hash, + struct buffer_head **bhp, + sector_t *secno, hash_t *next_hash); +int spadfs_lookup_ino(SPADFNODE *f, struct qstr *qstr, spadfs_ino_t *ino, + int for_delete); +int spadfs_check_directory_empty(SPADFNODE *f); +int spadfs_remove_directory(SPADFNODE *fn); +/* + * These two are returned by spadfs_add_fnode_to_directory (in addition of + * -ERROR or pointer on success). See the description above ND_LOCK. + */ +#define NEED_SYNC ((void *)1) +#define NEED_WLOCK ((void *)2) +struct fnode *spadfs_add_fnode_to_directory(SPADFNODE *dir, const char *name, + unsigned namelen, unsigned ea_size, + struct buffer_head **bhp, + sector_t *fnode_address, + unsigned *fnode_off, + struct fnode_block **pfnode_block, + int wlock); +int spadfs_remove_fnode_from_directory(SPADFNODE *dir, SPADFNODE *file, + struct qstr *name); +int spadfs_move_fnode_to_directory(SPADFNODE *old_dir, struct qstr *old_name, + SPADFNODE *old_file, SPADFNODE *new_dir, + struct qstr *new_name, + struct dentry *new_dentry, + u8 *new_ea, unsigned new_ea_size); + diff --git a/fs/spadfs/endian.h b/fs/spadfs/endian.h new file mode 100644 index 000000000..609ad4697 --- /dev/null +++ b/fs/spadfs/endian.h @@ -0,0 +1,7 @@ +#ifndef _SPADFS_COMMON_ENDIAN_H +#define _SPADFS_COMMON_ENDIAN_H + +#define SPADFS_LITTLE_ENDIAN +/*#define SPADFS_BIG_ENDIAN*/ + +#endif diff --git a/fs/spadfs/file.c b/fs/spadfs/file.c new file mode 100644 index 000000000..a38f393f9 --- /dev/null +++ b/fs/spadfs/file.c @@ -0,0 +1,1904 @@ +#include "spadfs.h" + +#define SPADFS_MPAGE + +#ifdef SPADFS_MPAGE +#include +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,15,0) +#include +#endif + +#define BMAP_EXTEND 0 +#define BMAP_MAP 1 + +static int file_make_memalloc(SPADFNODE *f, sector_t start, sector_t n_sectors) +{ + int r; + sector_t d_off_sink; + SPADFS *fs = f->fs; + mutex_lock(&fs->alloc_lock); + if (unlikely(spadfs_allocmem_find(fs, start, n_sectors, &d_off_sink))) { + spadfs_error(fs, 0, + "block range %Lx,%Lx does overlap with another memory-allocated block", + (unsigned long long)start, + (unsigned long long)n_sectors); + r = -EFSERROR; + goto unlock_ret; + } + r = spadfs_allocmem_add(fs, start, n_sectors); + if (unlikely(r)) { + printk(KERN_WARNING "spadfs: memory allocation failure, leaking disk space temporarily\n"); + r = 0; + goto unlock_ret; + } + r = spadfs_free_blocks_unlocked(fs, start, n_sectors); +unlock_ret: + mutex_unlock(&fs->alloc_lock); + return r; +} + +static int file_alloc_blocks(SPADFNODE *f, struct alloc *al) +{ + /*unsigned wanted = al->n_sectors;*/ + int r = spadfs_alloc_blocks(f->fs, al); + /*if (!r) printk("wanted %x, allocated: %x\n", wanted, al->n_sectors);*/ + if (likely(!r) && unlikely(is_deleted_file(f))) + r = file_make_memalloc(f, al->sector, al->n_sectors); + return r; +} + +static int file_free_blocks(SPADFNODE *f, sector_t start, sector_t n_sectors) +{ + int r; + SPADFS *fs = f->fs; + + mutex_lock(&fs->alloc_lock); + + if (!is_deleted_file(f)) + r = spadfs_free_blocks_unlocked(fs, start, n_sectors); + else { + do { + sector_t run_end; + r = spadfs_allocmem_find(fs, start, 1, &run_end); + if (!run_end || run_end > start + n_sectors) + run_end = start + n_sectors; + if (likely(r)) { + spadfs_allocmem_delete(fs, start, + run_end - start); + } else { + r = spadfs_free_blocks_unlocked(fs, start, + run_end - start); + if (unlikely(r)) + goto unlock_ret_r; + } + n_sectors -= run_end - start; + start = run_end; + } while (n_sectors); + r = 0; + } + +unlock_ret_r: + mutex_unlock(&fs->alloc_lock); + return r; +} + +sector_t spadfs_size_2_sectors(SPADFS *fs, loff_t size) +{ + sector_t result; + FILE_SECTORS(512U << fs->sectors_per_disk_block_bits, 512U << fs->sectors_per_cluster_bits, + fs->cluster_threshold, size, result); + return result; +} + +static void really_do_anode_checksum(struct anode *anode) +{ + anode->flags |= ANODE_CHECKSUM_VALID; + anode->checksum ^= CHECKSUM_BASE ^ __byte_sum(anode, ANODE_SIZE); +} + +static void do_anode_checksum(SPADFS *fs, struct anode *anode) +{ + if (unlikely(make_checksums(fs))) + really_do_anode_checksum(anode); + else + anode->flags &= ~ANODE_CHECKSUM_VALID; +} + +static struct anode *alloc_anode(SPADFNODE *f, sector_t hint, + struct buffer_head **bhp, sector_t *result) +{ + SPADFS *fs = f->fs; + int r; + struct alloc al; + struct anode *anode; + al.sector = hint; + al.n_sectors = 1U << fs->sectors_per_disk_block_bits; + al.extra_sectors = 0; + al.flags = ALLOC_METADATA; + al.reservation = NULL; + r = file_alloc_blocks(f, &al); + if (unlikely(r)) + return ERR_PTR(r); + *result = al.sector; + anode = spadfs_get_new_sector(fs, al.sector, bhp, "alloc_anode"); + if (unlikely(IS_ERR(anode))) + return anode; + memset(anode, 0, 512U << fs->sectors_per_buffer_bits); + CPU2SPAD32_LV(&anode->magic, spadfs_magic(fs, al.sector, ANODE_MAGIC)); + return anode; +} + +static void clear_extent_cache(SPADFNODE *f) +{ + unsigned i; + if (likely(spadfs_unlocked_extent_cache)) { + smp_wmb(); + WRITE_ONCE(f->extent_cache_seq, READ_ONCE(f->extent_cache_seq) + 1); + smp_wmb(); + } + for (i = 0; i < spadfs_extent_cache_size; i++) + WRITE_ONCE(f->extent_cache[i].n_sectors, 0); +} + +static int sync_bmap(SPADFNODE *f, sector_t lbn, sector_t *blk, sector_t *nblks, + sector_t *nback, int flags, sector_t *aptr, + const char *msg) +{ + int depth_now, depth_total; + sector_t ano; + sector_t nb; + struct anode *anode; + struct buffer_head *bh; + unsigned vx; + int direct, off; + + *aptr = 0; + + if (likely(lbn < f->blk1_n)) { + *blk = f->blk1 + lbn; + *nblks = f->blk1_n - lbn; + *nback = lbn; + if (unlikely(flags == BMAP_EXTEND)) { + f->blk1_n = lbn + 1; + f->blk2_n = 0; + f->root = 0; + clear_extent_cache(f); + } + return 0; + } + + if (likely(lbn < f->blk1_n + f->blk2_n)) { + *blk = f->blk2 + lbn - f->blk1_n; + *nblks = f->blk1_n + f->blk2_n - lbn; + *nback = lbn - f->blk1_n; + if (unlikely(flags == BMAP_EXTEND)) { + f->blk2_n = lbn - f->blk1_n + 1; + f->root = 0; + clear_extent_cache(f); + } + return 0; + } + + ano = f->root; + depth_now = 0; + depth_total = 0; + +subnode: + anode = spadfs_read_anode(f->fs, ano, &bh, &vx, flags != BMAP_EXTEND, + msg); + if (unlikely(IS_ERR(anode))) + return PTR_ERR(anode); + + direct = find_direct(depth_now, depth_total); + off = find_in_anode(anode, lbn, vx); + if (unlikely(lbn >= SPAD2CPU64_LV(&anode->x[off].end_off)) || + unlikely(lbn < SPAD2CPU64_LV(&anode->x[off - 1].end_off))) { + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "bmap(%s): out of range !(%Lx <= %Lx < %Lx), anode(%Lx -> %Lx)", + msg, + (unsigned long long)SPAD2CPU64_LV(&anode->x[off - 1].end_off), + (unsigned long long)lbn, + (unsigned long long)SPAD2CPU64_LV(&anode->x[off].end_off), + (unsigned long long)f->root, + (unsigned long long)ano); + goto brelse_err; + } + if (unlikely(off >= direct)) { + ano = SPAD2CPU64_LV(&anode->x[off].blk); + if (unlikely(flags == BMAP_EXTEND)) { + if (unlikely(anode->x[off].end_off != + CPU2SPAD64((u64)-1)) || + unlikely(anode->valid_extents != off + 1)) { + start_concurrent_atomic_buffer_modify(f->fs, + bh); + CPU2SPAD64_LV(&anode->x[off].end_off, (u64)-1); + anode->valid_extents = off + 1; + do_anode_checksum(f->fs, anode); + clear_extent_cache(f); + end_concurrent_atomic_buffer_modify(f->fs, bh); + } + } + update_depth(&depth_now, &depth_total, off); + if (flags != BMAP_EXTEND) + end_concurrent_atomic_buffer_read(f->fs, bh); + spadfs_brelse(f->fs, bh); + goto subnode; + } + + if (unlikely(!off) || (unlikely(off == 2) && unlikely(!depth_now))) + *aptr = ano; + + if (unlikely(SPAD2CPU64_LV(&anode->x[off].end_off) <= + SPAD2CPU64_LV(&anode->x[off - 1].end_off))) { + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "bmap(%s): non-monotonic anode (%Lx -> %Lx), entry %d", + msg, + (unsigned long long)f->root, + (unsigned long long)ano, + off); + goto brelse_err; + } + + nb = lbn - SPAD2CPU64_LV(&anode->x[off - 1].end_off); + *nback = nb; + *blk = SPAD2CPU64_LV(&anode->x[off].blk) + nb; + *nblks = SPAD2CPU64_LV(&anode->x[off].end_off) - lbn; + if (flags == BMAP_EXTEND) { + if (unlikely(SPAD2CPU64_LV(&anode->x[off].end_off) != lbn + 1) + || unlikely(anode->valid_extents != off + 1)) { + start_concurrent_atomic_buffer_modify(f->fs, bh); + CPU2SPAD64_LV(&anode->x[off].end_off, lbn + 1); + anode->valid_extents = off + 1; + do_anode_checksum(f->fs, anode); + clear_extent_cache(f); + end_concurrent_atomic_buffer_modify(f->fs, bh); + } + } + if (flags != BMAP_EXTEND) + end_concurrent_atomic_buffer_read(f->fs, bh); + spadfs_brelse(f->fs, bh); + return 0; + +brelse_err: + if (flags != BMAP_EXTEND) + end_concurrent_atomic_buffer_read(f->fs, bh); + spadfs_brelse(f->fs, bh); + return -EFSERROR; +} + +static int spadfs_add_extent(SPADFNODE *f, sector_t blk, sector_t n_blks) +{ + SPADFS *fs = f->fs; + struct anode *anode; + sector_t ano; + int depth_now, depth_total; + int direct; + sector_t ano_l; + sector_t end_off; + struct buffer_head *bh; + unsigned vx; + + if (!f->blk1_n) { + unsigned mdb; + f->blk1 = blk; + mdb = MAX_DIRECT_BLKS(1U << fs->sectors_per_disk_block_bits); + if (unlikely(n_blks > mdb)) { + f->blk1_n = mdb; + blk += mdb; + n_blks -= mdb; + goto again1; + } + f->blk1_n = n_blks; + return 0; + } + + if (!f->blk2_n) { + unsigned mdb; +again1: + f->blk2 = blk; + mdb = MAX_DIRECT_BLKS(1U << fs->sectors_per_disk_block_bits); + if (unlikely(n_blks > mdb)) { + f->blk2_n = mdb; + blk += mdb; + n_blks -= mdb; + goto again2; + } + f->blk2_n = n_blks; + return 0; + } + + if (!f->root) { +again2: + anode = alloc_anode(f, spadfs_alloc_hint(f, HINT_META), &bh, + &f->root); + if (unlikely(IS_ERR(anode))) + return PTR_ERR(anode); + + anode->flags |= ANODE_ROOT; + anode->valid_extents = 3; + CPU2SPAD64_LV(&anode->start_off, 0); + CPU2SPAD64_LV(&anode->x[0].blk, f->blk1); + CPU2SPAD64_LV(&anode->x[0].end_off, f->blk1_n); + CPU2SPAD64_LV(&anode->x[1].blk, f->blk2); + CPU2SPAD64_LV(&anode->x[1].end_off, f->blk1_n + f->blk2_n); + CPU2SPAD64_LV(&anode->x[2].blk, blk); + CPU2SPAD64_LV(&anode->x[2].end_off, f->blk1_n + f->blk2_n + n_blks); + do_anode_checksum(fs, anode); + mark_buffer_dirty(bh); + spadfs_brelse(fs, bh); + return 0; + } + + ano = f->root; + ano_l = -1; + depth_now = depth_total = 0; + +subnode: + anode = spadfs_read_anode(fs, ano, &bh, &vx, 0, "spadfs_add_extent 1"); + if (unlikely(IS_ERR(anode))) + return PTR_ERR(anode); + + direct = find_direct(depth_now, depth_total); + if (likely(vx < direct)) { + start_concurrent_atomic_buffer_modify(fs, bh); + CPU2SPAD64_LV(&anode->x[vx].blk, blk); + CPU2SPAD64_LV(&anode->x[vx].end_off, + SPAD2CPU64_LV(&anode->x[anode->valid_extents - 1].end_off) + + n_blks); + anode->valid_extents = vx + 1; + do_anode_checksum(fs, anode); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + return 0; + } + if (vx < ANODE_N_EXTENTS) + ano_l = ano; + if (vx > direct) { + ano = SPAD2CPU64_LV(&anode->x[vx - 1].blk); + update_depth(&depth_now, &depth_total, vx - 1); + spadfs_brelse(fs, bh); + goto subnode; + } + if (unlikely(ano_l == -1)) { + spadfs_brelse(fs, bh); + spadfs_error(fs, TXFLAGS_FS_ERROR, + "anode %Lx is full", + (unsigned long long)f->root); + return -EFSERROR; + } + + end_off = SPAD2CPU64_LV(&anode->x[vx - 1].end_off); + spadfs_brelse(fs, bh); + + anode = alloc_anode(f, spadfs_alloc_hint(f, HINT_META), &bh, &ano); + if (unlikely(IS_ERR(anode))) + return PTR_ERR(anode); + + anode->valid_extents = 1; + CPU2SPAD64_LV(&anode->start_off, end_off); + CPU2SPAD64_LV(&anode->x[0].blk, blk); + CPU2SPAD64_LV(&anode->x[0].end_off, end_off + n_blks); + do_anode_checksum(fs, anode); + mark_buffer_dirty(bh); + spadfs_brelse(fs, bh); + anode = spadfs_read_anode(fs, ano_l, &bh, &vx, 0, + "spadfs_add_extent 2"); + if (unlikely(IS_ERR(anode))) + return PTR_ERR(anode); + if (unlikely(vx == ANODE_N_EXTENTS)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, "anode %Lx filled under us", + (unsigned long long)ano_l); + spadfs_brelse(fs, bh); + return -EFSERROR; + } + start_concurrent_atomic_buffer_modify(fs, bh); + CPU2SPAD64_LV(&anode->x[vx].blk, ano); + CPU2SPAD64_LV(&anode->x[vx].end_off, (u64)-1); + CPU2SPAD64_LV(&anode->x[vx - 1].end_off, end_off); + anode->valid_extents = vx + 1; + do_anode_checksum(fs, anode); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + return 0; +} + +static int spadfs_extend_last_extent(SPADFNODE *f, sector_t n_blks) +{ + SPADFS *fs = f->fs; + struct anode *anode; + sector_t ano; + int depth_now, depth_total; + int direct; + struct buffer_head *bh; + unsigned vx; + + if (!f->blk2_n) { + unsigned mdb = MAX_DIRECT_BLKS(1U << fs->sectors_per_disk_block_bits); + if (f->blk1_n + n_blks > mdb) { + n_blks -= mdb - f->blk1_n; + f->blk1_n = mdb; + return spadfs_add_extent(f, f->blk1 + mdb, n_blks); + } + f->blk1_n += n_blks; + return 0; + } + + if (!f->root) { + unsigned mdb = MAX_DIRECT_BLKS(1U << fs->sectors_per_disk_block_bits); + if (f->blk2_n + n_blks > mdb) { + n_blks -= mdb - f->blk2_n; + f->blk2_n = mdb; + return spadfs_add_extent(f, f->blk2 + mdb, n_blks); + } + f->blk2_n += n_blks; + return 0; + } + ano = f->root; + depth_now = depth_total = 0; + +subnode: + anode = spadfs_read_anode(fs, ano, &bh, &vx, 0, + "spadfs_extend_last_extent"); + if (unlikely(IS_ERR(anode))) + return PTR_ERR(anode); + + direct = find_direct(depth_now, depth_total); + if (vx <= direct) { + start_concurrent_atomic_buffer_modify(fs, bh); + CPU2SPAD64_LV(&anode->x[vx - 1].end_off, SPAD2CPU64_LV(&anode->x[vx - 1].end_off) + n_blks); + do_anode_checksum(fs, anode); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + return 0; + } + + ano = SPAD2CPU64_LV(&anode->x[vx - 1].blk); + update_depth(&depth_now, &depth_total, vx - 1); + spadfs_brelse(fs, bh); + goto subnode; +} + +static int spadfs_do_truncate_blocks(SPADFNODE *f, sector_t blks, sector_t oldblks) +{ + SPADFS *fs = f->fs; + sector_t d_off_sink; /* write-only */ + int r; + sector_t blk, bback, ano; + + spadfs_discard_reservation(fs, &f->res); + + if (!oldblks) + return 0; + +again: + r = sync_bmap(f, blks - 1, &blk, &d_off_sink, &bback, BMAP_MAP, &ano, + "spadfs_do_truncate_blocks"); + if (unlikely(r)) + return r; + + blk++; + bback++; + if (oldblks >= bback && unlikely(ano != 0)) { + r = file_free_blocks(f, ano, 1U << fs->sectors_per_disk_block_bits); + if (unlikely(r)) + return r; + } + + if (oldblks < bback) + bback = oldblks; + r = file_free_blocks(f, blk - bback, bback); + if (unlikely(r)) + return r; + + blks -= bback; + if ((oldblks -= bback)) + goto again; + return 0; +} + +void spadfs_create_memory_extents(SPADFNODE *f) +{ + SPADFS *fs = f->fs; + sector_t lbn = 0; + sector_t total_lbns = spadfs_size_2_sectors(fs, f->disk_size); + sector_t d_off_sink; + sector_t blk, blks, ano; + int r; + + while (lbn < total_lbns) { + r = sync_bmap(f, lbn, &blk, &blks, &d_off_sink, BMAP_MAP, &ano, + "spadfs_create_memory_extents"); + if (unlikely(r)) + return; + if (blks > total_lbns - lbn) + blks = total_lbns - lbn; + r = file_make_memalloc(f, blk, blks); + if (unlikely(r)) + return; + if (unlikely(ano != 0)) { + r = file_make_memalloc(f, ano, 1U << fs->sectors_per_disk_block_bits); + if (unlikely(r)) + return; + } + lbn += blks; + } +} + +static void spadfs_set_disk_size(SPADFNODE *f, loff_t newsize) +{ + if (f->commit_sequence != f->fs->commit_sequence) { + f->crash_disk_size = f->disk_size; + f->commit_sequence = f->fs->commit_sequence; + } + f->disk_size = newsize; +} + +static int spadfs_do_truncate(SPADFNODE *f, loff_t newsize) +{ + SPADFS *fs = f->fs; + sector_t blks, oldblks, nsb; + blks = spadfs_size_2_sectors(fs, f->disk_size); + nsb = spadfs_size_2_sectors(fs, newsize); + oldblks = blks - nsb; +#ifdef SPADFS_QUOTA + dquot_free_space_nodirty(inode(f), (loff_t)oldblks << 9); +#else + inode_sub_bytes(inode(f), (loff_t)oldblks << 9); +#endif + spadfs_set_disk_size(f, spadfs_roundup_blocksize(fs, newsize)); + return spadfs_do_truncate_blocks(f, blks, oldblks); +} + +static int spadfs_do_extend(SPADFNODE *f, loff_t size, unsigned extra_sectors, sector_t resurrect_blks) +{ + SPADFS *fs = f->fs; + struct alloc al; + sector_t d_off_sink; /* write-only */ + sector_t blks, newblks, blks_to_do; + sector_t blk; + int flags; + int r; + + BUG_ON((unsigned long)size & ((512U << fs->sectors_per_disk_block_bits) - 1)); + + blks = spadfs_size_2_sectors(fs, f->disk_size); + newblks = spadfs_size_2_sectors(fs, size) - blks; + blks_to_do = newblks; + +#ifdef SPADFS_QUOTA + r = dquot_alloc_space_nodirty(inode(f), (loff_t)newblks << 9); + if (unlikely(r)) + goto ret_r; +#else + inode_add_bytes(inode(f), (loff_t)newblks << 9); +#endif + +#ifdef SPADFS_RESURRECT + while (unlikely(resurrect_blks != 0) && blks_to_do) { + sector_t nfwd, nback, ano; + r = sync_bmap(f, blks, &blk, &nfwd, &nback, + BMAP_MAP, &ano, "spadfs_do_extend (resurrect)"); + if (unlikely(r)) + goto unaccount_ret_r; + + if (nfwd > resurrect_blks) + nfwd = resurrect_blks; + if (nfwd > blks_to_do) + nfwd = blks_to_do; + + if (unlikely(!nback) && ano != 0) { + al.sector = ano; + al.n_sectors = 1U << fs->sectors_per_disk_block_bits; + al.extra_sectors = 0; + al.flags = ALLOC_RESURRECT; + al.reservation = NULL; + r = spadfs_alloc_blocks(fs, &al); + if (unlikely(r)) + goto unaccount_ret_r; + } + al.sector = blk; + al.n_sectors = nfwd; + al.extra_sectors = 0; + al.flags = ALLOC_RESURRECT; + al.reservation = NULL; + r = spadfs_alloc_blocks(fs, &al); + if (unlikely(r)) + goto unaccount_ret_r; + + resurrect_blks -= al.n_sectors; + blks_to_do -= al.n_sectors; + blks += al.n_sectors; + } +#endif + + if (!blks_to_do) + goto set_size_return; + + flags = size + (512U << fs->sectors_per_disk_block_bits) > fs->cluster_threshold ? + ALLOC_BIG_FILE : ALLOC_SMALL_FILE; + + /*printk("size %Lx, disk_size %Lx, blks %Lx\n", (unsigned long long)size, (unsigned long long)f->disk_size, (unsigned long long)0);*/ + + if (blks) { + r = sync_bmap(f, blks - 1, &blk, &d_off_sink, &d_off_sink, + BMAP_EXTEND, &d_off_sink, "spadfs_do_extend"); + if (unlikely(r)) + goto unaccount_ret_r; + blk++; + } else { + clear_extent_cache(f); + blk = 0; + } + + if (blks && + (flags == ALLOC_SMALL_FILE || + likely(f->disk_size > fs->cluster_threshold))) { +new_run: + al.n_sectors = blks_to_do; + al.extra_sectors = extra_sectors; + al.flags = flags | ALLOC_PARTIAL_AT_GOAL; +alloc_c: + al.reservation = &f->res; + al.sector = blk; + /*printk("cnt: bl %Lx, hi %Lx, ns %x, fl %x", (unsigned long long)blks, (unsigned long long)al.sector, al.n_sectors, al.flags);*/ + r = file_alloc_blocks(f, &al); + if (unlikely(r)) { + /*printk(" -> error %d\n", r);*/ + goto truncate_ret_r; + } + /*printk(" -> %Lx, %x\n", (unsigned long long)al.sector, al.n_sectors);*/ + } else { + if (blks) { +new_extent: + if ((unsigned long)blks & + ((1U << fs->sectors_per_cluster_bits) - 1)) { + flags = 0; + al.flags = ALLOC_SMALL_FILE | ALLOC_PARTIAL_AT_GOAL; + al.n_sectors = + (1U << fs->sectors_per_cluster_bits) - + ((unsigned long)blks & + ((1U << fs->sectors_per_cluster_bits) - + 1)); + if (al.n_sectors > blks_to_do) + al.n_sectors = blks_to_do; + al.extra_sectors = 0; + goto alloc_c; + } + if (blk >= (sector_t)fs->zones[2].grp_start << fs->sectors_per_group_bits) + goto new_run; + } else { + f->blk1_n = 0; + f->blk2_n = 0; + f->root = 0; + } + + al.sector = spadfs_alloc_hint(f, + flags & ALLOC_BIG_FILE ? HINT_BIG : HINT_SMALL); + al.n_sectors = blks_to_do; + al.extra_sectors = extra_sectors; + al.flags = flags; + al.reservation = &f->res; + /*printk("new: bl %Lx, hi %Lx, ns %x, fl %x", (unsigned long long)blks, (unsigned long long)al.sector, al.n_sectors, al.flags);*/ + r = file_alloc_blocks(f, &al); + if (unlikely(r)) { + /*printk(" -> error %d\n", r);*/ + goto truncate_ret_r; + } + /*printk(" -> %Lx, %x\n", (unsigned long long)al.sector, al.n_sectors);*/ + + if (unlikely(al.flags & ALLOC_NEW_GROUP_HINT)) + spadfs_set_new_hint(f, &al); + } + + if (likely(al.sector == blk)) + r = spadfs_extend_last_extent(f, al.n_sectors); + else + r = spadfs_add_extent(f, al.sector, al.n_sectors); + + if (unlikely(r)) { + int rr = file_free_blocks(f, al.sector, al.n_sectors); + if (unlikely(rr)) + r = rr; + goto truncate_ret_r; + } + + blks_to_do -= al.n_sectors; + blks += al.n_sectors; + + if (unlikely(blks_to_do != 0)) { + blk = al.sector + al.n_sectors; + if (unlikely(!flags)) { + flags = ALLOC_BIG_FILE; + goto new_extent; + } else + goto new_run; + } + +set_size_return: + smp_wmb(); + spadfs_set_disk_size(f, size); + return 0; + +truncate_ret_r: + spadfs_do_truncate_blocks(f, blks, + blks - spadfs_size_2_sectors(fs, f->disk_size)); + +unaccount_ret_r: +#ifdef SPADFS_QUOTA + dquot_free_space_nodirty(inode(f), (loff_t)newblks << 9); +#else + inode_sub_bytes(inode(f), (loff_t)newblks << 9); +#endif + +#ifdef SPADFS_QUOTA +ret_r: +#endif + return r; +} + +static void spadfs_get_blocks_to_resurrect(SPADFNODE *fn, sector_t *result) +{ + SPADFS *fs = fn->fs; + *result = 0; + if (fn->commit_sequence == fs->commit_sequence && + unlikely(fn->crash_disk_size > fn->disk_size)) { + sector_t dblks = spadfs_size_2_sectors(fs, fn->crash_disk_size); + sector_t sblks = spadfs_size_2_sectors(fs, fn->disk_size); + if (unlikely(dblks > sblks)) + *result = dblks - sblks; + } +} + +static noinline int spadfs_extend_file(struct inode *i, sector_t target_blocks, int target_blocks_exact) +{ + sync_lock_decl + SPADFS *fs = spadfnode(i)->fs; + int synced = 0; + loff_t needed_size; + unsigned long long long_prealloc; + unsigned prealloc; + unsigned ts; + int r; + sector_t resurrect_blks; + +retry: + down_read_sync_lock(fs); + mutex_lock(&spadfnode(i)->file_lock); + + spadfs_get_blocks_to_resurrect(spadfnode(i), &resurrect_blks); + +#ifndef SPADFS_RESURRECT + if (unlikely(resurrect_blks != 0)) + goto unlock_commit; +#endif + + needed_size = spadfnode(i)->disk_size + (512U << fs->sectors_per_disk_block_bits); + + long_prealloc = (((unsigned long long)target_blocks) << (fs->sectors_per_disk_block_bits + 9)) - needed_size; + if (unlikely((long long)long_prealloc < 0)) + long_prealloc = 0; + ts = 0; + if (spadfnode(i)->disk_size >= fs->cluster_threshold && likely(!target_blocks_exact)) { + uint64_t ts64; + uint64_t ds = spadfnode(i)->disk_size; + if (likely(fs->prealloc_part_bits >= 0)) + ds >>= fs->prealloc_part_bits; + else + do_div(ds, fs->prealloc_part); + if (ds < fs->min_prealloc) + ds = fs->min_prealloc; + ts64 = ds >> 9; + if (unlikely(ts64 >= 0x100000000ULL)) + ts = -1U; + else + ts = ts64; + if (ds > fs->max_prealloc) + ds = fs->max_prealloc; + if (long_prealloc < (unsigned)ds) + long_prealloc = (unsigned)ds; + } + + /*{ + unsigned long long max_allocation; + max_allocation = (unsigned long long)READ_ONCE(fs->max_allocation) * 512; + if (likely(max_allocation >= 512U << fs->sectors_per_disk_block_bits)) + max_allocation -= 512U << fs->sectors_per_disk_block_bits; + if (unlikely(long_prealloc > max_allocation)) + long_prealloc = max_allocation; + }*/ + + { + long long freespace_limit = (unsigned long long)READ_ONCE(fs->freespace) * 512 - ((512U << fs->sectors_per_disk_block_bits) + (512U << fs->sectors_per_cluster_bits)); + if (unlikely((long long)long_prealloc > freespace_limit)) { + if (freespace_limit < 0) + long_prealloc = 0; + else + long_prealloc = freespace_limit; + } + } + + if (unlikely(long_prealloc > (u32)-(1024U << fs->sectors_per_disk_block_bits))) { + unsigned long long remaining = long_prealloc - (u32)-(1024U << fs->sectors_per_disk_block_bits); + remaining = (remaining + (512U << fs->sectors_per_disk_block_bits) - 1) >> 9; + if (remaining >= 0x100000000ULL) + ts = -1U; + else if ((unsigned)remaining > ts) + ts = remaining; + prealloc = (u32)-(1024U << fs->sectors_per_disk_block_bits); + } else { + prealloc = long_prealloc; + } + prealloc &= ~((512U << fs->sectors_per_disk_block_bits) - 1); + + if (unlikely(!spadfnode(i)->ea_unx)) { + prealloc >>= 9; + if (likely(prealloc + ts >= ts)) + ts += prealloc; + else + ts = -1U; + prealloc = 0; + } + ts &= -(1U << fs->sectors_per_disk_block_bits); + +again_without_prealloc: + /*printk("prealloc: %llx + %x = %llx\n", needed_size, prealloc, needed_size + prealloc);*/ + r = spadfs_do_extend(spadfnode(i), needed_size + prealloc, ts, resurrect_blks); + if (unlikely(r)) { + if (likely(r == -ENOSPC)) { + if (prealloc && target_blocks_exact != 2) { + prealloc = 0; + goto again_without_prealloc; + } + if (!synced) + goto unlock_commit; + } + goto unlock_return_r; + } + + spadfs_write_file(spadfnode(i), 0, NULL, NULL); + + r = 0; + + if (likely(target_blocks_exact != 2)) { + if (unlikely(READ_ONCE(spadfnode(i)->dont_truncate_prealloc))) + spadfnode(i)->dont_truncate_prealloc = 0; + } else { + spadfnode(i)->dont_truncate_prealloc = 1; + } + +unlock_return_r: + mutex_unlock(&spadfnode(i)->file_lock); + up_read_sync_lock(fs); + return r; + +unlock_commit: + mutex_unlock(&spadfnode(i)->file_lock); + up_read_sync_lock(fs); + + if (unlikely(sb_rdonly(fs->s))) + return -EROFS; + + r = spadfs_commit(fs); + if (unlikely(r)) + return r; + + synced = 1; + goto retry; +} + +static int spadfs_inode_needs_clear(SPADFS *fs, struct inode *i) +{ + return (spadfnode(i)->clear_position & ((1U << fs->sectors_per_disk_block_bits) - 1)) != 0; +} + +static noinline void spadfs_add_inode_to_clear_list(struct inode *i) +{ + SPADFS *fs = spadfnode(i)->fs; + spadfnode(i)->clear_position = (sector_t)((spadfnode(i)->mmu_private + (512U << fs->sectors_per_buffer_bits) - 1) >> (fs->sectors_per_buffer_bits + 9)) << fs->sectors_per_buffer_bits; + if (unlikely(list_empty(&spadfnode(i)->clear_entry)) && spadfs_inode_needs_clear(fs, i)) { + spin_lock(&fs->clear_lock); + list_add(&spadfnode(i)->clear_entry, &fs->clear_list); + spin_unlock(&fs->clear_lock); + } +} + +void spadfs_clear_last_block(struct inode *i) +{ + SPADFNODE *f = spadfnode(i); + SPADFS *fs = f->fs; + spin_lock(&fs->clear_lock); + list_del_init(&f->clear_entry); + spin_unlock(&fs->clear_lock); + if (spadfs_inode_needs_clear(fs, i)) { + sector_t result, d_off_sink; + int r; + r = sync_bmap(f, f->clear_position, &result, &d_off_sink, &d_off_sink, BMAP_MAP, &d_off_sink, "spadfs_clear_last_block"); + if (unlikely(r)) + return; + do { + struct buffer_head *bh; + void *data = spadfs_get_new_sector(fs, result, &bh, "spadfs_clear_last_block"); + if (unlikely(IS_ERR(data))) + break; + memset(data, 0, 512U << fs->sectors_per_buffer_bits); + mark_buffer_dirty(bh); + spadfs_brelse(fs, bh); + result += 1U << fs->sectors_per_buffer_bits; + } while (result & ((1U << fs->sectors_per_disk_block_bits) - 1)); + } +} + +static inline void spadfs_test_and_clear_last_block(struct inode *i) +{ + if (unlikely(!list_empty(&spadfnode(i)->clear_entry))) + spadfs_clear_last_block(i); +} + +static int spadfs_get_block(struct inode *i, sector_t lblock, struct buffer_head *bh_result, int create) +{ + sector_t n_fwd, n_back; + sector_t d_off_sink; /* write-only */ + int r; + sector_t result; + u64 extent_cache_seq; + struct extent_cache *x; + SPADFS *fs = spadfnode(i)->fs; + sector_t blk = (sector_t)lblock << fs->sectors_per_buffer_bits; + + if (unlikely(create)) { + if (likely(blk >= (spadfnode(i)->mmu_private + 511) >> 9)) { + set_buffer_new(bh_result); + BUG_ON((loff_t)blk << 9 != spadfnode(i)->mmu_private); + BUG_ON(spadfnode(i)->mmu_private > spadfnode(i)->disk_size); + if (unlikely(spadfnode(i)->mmu_private == spadfnode(i)->disk_size)) { + r = spadfs_extend_file(i, READ_ONCE(spadfnode(i)->target_blocks), READ_ONCE(spadfnode(i)->target_blocks_exact)); + if (unlikely(r)) + return r; + } + spadfnode(i)->mmu_private += 512U << fs->sectors_per_buffer_bits; + if (unlikely(fs->sectors_per_buffer_bits != fs->sectors_per_disk_block_bits)) { + sync_lock_decl + down_read_sync_lock(fs); + spadfs_add_inode_to_clear_list(i); + up_read_sync_lock(fs); + } + } + } + + if (likely(spadfs_unlocked_extent_cache)) { + preempt_disable(); + x = &spadfnode(i)->extent_cache[smp_processor_id()]; + } else { + mutex_lock(&spadfnode(i)->file_lock); + x = &spadfnode(i)->extent_cache[0]; + } + + if (likely(blk >= READ_ONCE(x->logical_sector)) && + likely(blk < READ_ONCE(x->logical_sector) + READ_ONCE(x->n_sectors))) { + sector_t o = blk - READ_ONCE(x->logical_sector); + n_fwd = READ_ONCE(x->n_sectors) - o; + result = READ_ONCE(x->physical_sector) + o; + goto preempt_en_ret_result; + } + + extent_cache_seq = 0; /* against warning */ + if (likely(spadfs_unlocked_extent_cache)) { + preempt_enable(); + extent_cache_seq = READ_ONCE(spadfnode(i)->extent_cache_seq); + smp_mb(); + } + + r = sync_bmap(spadfnode(i), blk, &result, &n_fwd, &n_back, BMAP_MAP, &d_off_sink, "spadfs_get_block"); + + if (unlikely(r)) { + if (unlikely(!spadfs_unlocked_extent_cache)) + mutex_unlock(&spadfnode(i)->file_lock); + return r; + } + + if (likely(spadfs_unlocked_extent_cache)) { + preempt_disable(); + x = &spadfnode(i)->extent_cache[smp_processor_id()]; + } + + WRITE_ONCE(x->physical_sector, result - n_back); + WRITE_ONCE(x->logical_sector, blk - n_back); + /* truncate it from sector_t to unsigned long */ + WRITE_ONCE(x->n_sectors, n_back + n_fwd); + + if (likely(spadfs_unlocked_extent_cache)) { + smp_mb(); + if (unlikely(READ_ONCE(spadfnode(i)->extent_cache_seq) != extent_cache_seq)) + WRITE_ONCE(x->n_sectors, 0); + } + +preempt_en_ret_result: + if (likely(spadfs_unlocked_extent_cache)) + preempt_enable(); + else + mutex_unlock(&spadfnode(i)->file_lock); + + if (bh_result->b_size >> 9 < n_fwd) + n_fwd = bh_result->b_size >> 9; + map_bh(bh_result, i->i_sb, result >> fs->sectors_per_buffer_bits); + bh_result->b_size = (size_t)n_fwd << 9; + return 0; +} + +#ifdef SPADFS_DIRECT_IO + +/* Direct I/O sometimes sends requests beyond file end */ + +static int spadfs_get_block_direct(struct inode *i, sector_t lblock, struct buffer_head *bh_result, int create) +{ + int r; + sector_t blk, total_blocks; + SPADFS *fs = spadfnode(i)->fs; + + BUG_ON(!inode_is_locked(i)); + + blk = (sector_t)lblock << fs->sectors_per_buffer_bits; + if (!create) { + if (unlikely(blk >= (spadfnode(i)->mmu_private + 511) >> 9)) + return 0; + } else { + if (unlikely(blk > spadfnode(i)->mmu_private >> 9)) + return 0; + } + + r = spadfs_get_block(i, lblock, bh_result, create); + if (unlikely(r) || unlikely(!buffer_mapped(bh_result))) + return r; + + lblock += bh_result->b_size >> (9 + fs->sectors_per_buffer_bits); + total_blocks = (spadfnode(i)->mmu_private + ((512U << fs->sectors_per_buffer_bits) - 1)) >> (9 + fs->sectors_per_buffer_bits); + if (unlikely(lblock > total_blocks)) + bh_result->b_size -= (lblock - total_blocks) << (9 + fs->sectors_per_buffer_bits); + + return 0; +} + +#endif + +static void spadfs_discard_prealloc(SPADFNODE *f) +{ + loff_t inode_size = i_size_read(inode(f)); + if (inode_size + (512U << f->fs->sectors_per_disk_block_bits) <= f->disk_size) + spadfs_do_truncate(f, inode_size); + spadfs_write_file(f, 0, NULL, NULL); +} + +static void set_target_size(SPADFNODE *f, loff_t size, int exact) +{ + if (size < i_size_read(inode(f))) + return; + WRITE_ONCE(f->target_blocks, ((unsigned long long)size + ((512U << f->fs->sectors_per_disk_block_bits) - 1)) >> (f->fs->sectors_per_disk_block_bits + 9)); + WRITE_ONCE(f->target_blocks_exact, exact); +} + +static void spadfs_truncate_unlocked(struct inode *i) +{ + loff_t newsize = i->i_size; + + set_target_size(spadfnode(i), newsize, 1); + + block_truncate_page(i->i_mapping, newsize, spadfs_get_block); + + BUG_ON(newsize > spadfnode(i)->mmu_private); + BUG_ON(newsize > spadfnode(i)->disk_size); + spadfnode(i)->mmu_private = newsize; + spadfs_do_truncate(spadfnode(i), newsize); + spadfs_write_file(spadfnode(i), 0, NULL, NULL); + + if (unlikely(spadfnode(i)->fs->sectors_per_buffer_bits != spadfnode(i)->fs->sectors_per_disk_block_bits)) { + spadfs_add_inode_to_clear_list(i); + spadfs_test_and_clear_last_block(i); + } +} + +void spadfs_truncate(struct inode *i) +{ + sync_lock_decl + SPADFS *fs = spadfnode(i)->fs; + + down_read_sync_lock(fs); + mutex_lock(&spadfnode(i)->file_lock); + + spadfs_truncate_unlocked(i); + + mutex_unlock(&spadfnode(i)->file_lock); + up_read_sync_lock(fs); +} + +static int spadfs_release(struct inode *i, struct file *file) +{ + if (unlikely(file->f_mode & FMODE_WRITE)) { + sync_lock_decl + SPADFS *fs = spadfnode(i)->fs; + + spadfs_discard_reservation(fs, &spadfnode(i)->res); + + if (!inode_trylock(i)) { + /* + * This can only happen if the file is still open. + * In this case, don't cleanup prealloc. + * This mutex_trylock prevents a deadlock in sys_swapon + * when the swapfile is invalid. + * + * The last block must be always cleared, so we take + * filesystem sync lock if we couldn't take inode lock. + */ + + if (unlikely(!list_empty(&spadfnode(i)->clear_entry))) { + down_write_sync_lock(fs); + spadfs_test_and_clear_last_block(i); + up_write_sync_lock(fs); + } + + goto dont_cleanup; + } + down_read_sync_lock(fs); + mutex_lock(&spadfnode(i)->file_lock); + + if (likely(!spadfnode(i)->dont_truncate_prealloc)) + spadfs_discard_prealloc(spadfnode(i)); + + spadfs_test_and_clear_last_block(i); + + mutex_unlock(&spadfnode(i)->file_lock); + up_read_sync_lock(fs); + inode_unlock(i); + } + +dont_cleanup: + return 0; +} + +void spadfs_delete_file_content(SPADFNODE *f) +{ + spadfs_do_truncate(f, 0); +} + +#ifndef SPADFS_MPAGE + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0) +static int spadfs_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, spadfs_get_block); +} +#else +static int spadfs_readpage(struct file *file, struct page *page) +{ + return block_read_full_page(page, spadfs_get_block); +} +#endif + +static int spadfs_writepage(struct page *page, struct writeback_control *wbc) +{ + return block_write_full_page(page, spadfs_get_block, wbc); +} + +#else + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0) +static int spadfs_read_folio(struct file *file, struct folio *folio) +{ + return mpage_read_folio(folio, spadfs_get_block); +} +#else +static int spadfs_readpage(struct file *file, struct page *page) +{ + return mpage_readpage(page, spadfs_get_block); +} +#endif + +static int spadfs_writepage(struct page *page, struct writeback_control *wbc) +{ + /*return mpage_writepage(page, spadfs_get_block, wbc);*/ + return block_write_full_page(page, spadfs_get_block, wbc); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,8,0) || TEST_RHEL_VERSION(8,7) +static void spadfs_readahead(struct readahead_control *rac) +{ + mpage_readahead(rac, spadfs_get_block); +} +#else +static int spadfs_readpages(struct file *file, struct address_space *mapping, + struct list_head *pages, unsigned nr_pages) +{ + return mpage_readpages(mapping, pages, nr_pages, spadfs_get_block); +} +#endif + +static int spadfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, spadfs_get_block); +} + +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) +static inline void spadfs_write_failed(struct address_space *mapping, loff_t to) +{ +} +#else +static void spadfs_write_failed(struct address_space *mapping, loff_t to) +{ + sync_lock_decl + struct inode *i = mapping->host; + SPADFS *fs = spadfnode(i)->fs; + + down_read_sync_lock(fs); + mutex_lock(&spadfnode(i)->file_lock); + + if (to > i->i_size) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,12,0) && !TEST_RHEL_VERSION(7,1) + truncate_pagecache(i, to, i->i_size); +#else + truncate_pagecache(i, i->i_size); +#endif + spadfs_truncate_unlocked(i); + } + + mutex_unlock(&spadfnode(i)->file_lock); + up_read_sync_lock(fs); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) +static int spadfs_prepare_write(struct file *file, struct page *page, + unsigned from, unsigned to) +{ + SPADFNODE *f = spadfnode(page->mapping->host); + return cont_prepare_write(page, from, to, spadfs_get_block, &f->mmu_private); +} +#else +static int spadfs_write_begin(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,19,0) + unsigned flags, +#endif + struct page **pagep, void **fsdata) +{ + SPADFNODE *f = spadfnode(mapping->host); + int r; + + *pagep = NULL; +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,19,0) + r = cont_write_begin(file, mapping, pos, len, flags, pagep, fsdata, spadfs_get_block, &f->mmu_private); +#else + r = cont_write_begin(file, mapping, pos, len, pagep, fsdata, spadfs_get_block, &f->mmu_private); +#endif + + if (unlikely(r < 0)) { + spadfs_write_failed(mapping, pos + len); + } + + return r; +} +static int spadfs_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) +{ + int r; + + r = generic_write_end(file, mapping, pos, len, copied, page, fsdata); + + if (unlikely(r < 0) || unlikely(r < len)) { + spadfs_write_failed(mapping, pos + len); + } + + return r; +} +#endif + +static int spadfs_get_block_bmap(struct inode *i, sector_t lblock, struct buffer_head *bh_result, int create) +{ + SPADFS *fs = spadfnode(i)->fs; + sector_t num_blocks; + int r; + + num_blocks = spadfs_size_2_sectors(fs, spadfnode(i)->disk_size) >> fs->sectors_per_buffer_bits; + if (lblock >= num_blocks) + return 0; + + bh_result->b_size = -(size_t)(512U << fs->sectors_per_buffer_bits); + + r = spadfs_get_block(i, lblock, bh_result, 0); + if (unlikely(r) || unlikely(!buffer_mapped(bh_result))) + return r; + + if (bh_result->b_size >> (fs->sectors_per_buffer_bits + 9) >= num_blocks - lblock) + bh_result->b_size = (num_blocks - lblock) << (fs->sectors_per_buffer_bits + 9); + + return 0; +} + +static sector_t spadfs_bmap(struct address_space *mapping, sector_t block) +{ + sync_lock_decl + SPADFNODE *f = spadfnode(mapping->host); + SPADFS *fs = f->fs; + sector_t result; + + spadfs_cond_resched(); + + /* + * The kernel doesn't synchronize the bmap call with anything, so we + * must do synchronization on our own. + */ + if (likely(spadfs_unlocked_extent_cache)) { + down_read_sync_lock(fs); + mutex_lock(&f->file_lock); + } else { + down_write_sync_lock(fs); + } + + result = generic_block_bmap(mapping, block, spadfs_get_block_bmap); + + if (likely(spadfs_unlocked_extent_cache)) { + mutex_unlock(&f->file_lock); + up_read_sync_lock(fs); + } else { + up_write_sync_lock(fs); + } + + return result; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,15,0) +static int spadfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned flags, struct iomap *iomap, struct iomap *srcmap) +{ + unsigned int blkbits = inode->i_blkbits; + int r; + struct buffer_head tmp = { + .b_size = 1 << blkbits, + }; + + spadfs_cond_resched(); + + r = spadfs_get_block_bmap(inode, offset >> blkbits, &tmp, 0); + if (unlikely(r)) + return r; + + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = offset; + if (!buffer_mapped(&tmp)) { + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + iomap->length = 1 << blkbits; + } else { + iomap->type = IOMAP_MAPPED; + iomap->flags = IOMAP_F_MERGED; + iomap->addr = (u64)tmp.b_blocknr << blkbits; + iomap->length = tmp.b_size; + } + return 0; +} + +static const struct iomap_ops spadfs_iomap_ops = { + .iomap_begin = spadfs_iomap_begin, +}; +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,28) || TEST_RHEL_VERSION(5,4) +int spadfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,15,0) + int ret; + inode_lock(inode); + len = min_t(u64, len, i_size_read(inode)); + ret = iomap_fiemap(inode, fieinfo, start, len, &spadfs_iomap_ops); + inode_unlock(inode); + return ret; +#else + return generic_block_fiemap(inode, fieinfo, start, len, spadfs_get_block_bmap); +#endif +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,1,0) +static ssize_t spadfs_file_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + ssize_t r; + SPADFNODE *f = spadfnode(file_inode(file)); + + /* This is just advisory, so it needs no locking */ + set_target_size(f, *ppos + count, 0); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) + r = generic_file_write(file, buf, count, ppos); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3,16,0) + r = do_sync_write(file, buf, count, ppos); +#else + r = new_sync_write(file, buf, count, ppos); +#endif + return r; +} +#else +static ssize_t spadfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + SPADFNODE *f = spadfnode(file_inode(iocb->ki_filp)); + + /* This is just advisory, so it needs no locking */ + set_target_size(f, iocb->ki_pos + from->count, 0); + + return generic_file_write_iter(iocb, from); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35) +static int spadfs_file_fsync(struct file *file, struct dentry *dentry, + int datasync) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0) +static int spadfs_file_fsync(struct file *file, int datasync) +#else +static int spadfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) +#endif +{ + int r; + int optimized; + struct buffer_head *bh; + sync_lock_decl + SPADFNODE *f = spadfnode(file_inode(file)); + SPADFS *fs = f->fs; + + if (unlikely(sb_rdonly(fs->s))) + return 0; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,1,0) + r = filemap_write_and_wait_range(inode(f)->i_mapping, start, end); + if (unlikely(r)) + return r; +#endif + + down_read_sync_lock(fs); + mutex_lock(&f->file_lock); + optimized = 0; + bh = NULL; + r = spadfs_write_file(f, datasync, &optimized, &bh); + mutex_unlock(&f->file_lock); + up_read_sync_lock(fs); + + if (unlikely(r)) { + if (bh) + spadfs_brelse(fs, bh); + return r; + } + + if (likely(optimized)) { + if (likely(bh != NULL)) { + r = spadfs_sync_dirty_buffer(bh); + spadfs_brelse(fs, bh); + if (unlikely(r)) + return r; + r = spadfs_issue_flush(fs); + if (unlikely(r)) + return r; + } + return 0; + } + + if (unlikely(bh != NULL)) + spadfs_brelse(fs, bh); + + return spadfs_commit(fs); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) +int spadfs_file_setattr(struct dentry *dentry, struct iattr *iattr) +#else +int spadfs_file_setattr(struct mnt_idmap *ns, struct dentry *dentry, struct iattr *iattr) +#endif +{ + sync_lock_decl + struct inode *inode = dentry->d_inode; + int r; + if (iattr->ia_valid & ATTR_SIZE) { + if (unlikely(iattr->ia_size > inode->i_size)) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,16) + set_target_size(spadfnode(inode), iattr->ia_size, 1); + r = generic_cont_expand_simple(inode, iattr->ia_size); + if (unlikely(r)) + return r; +#else + return -EPERM; +#endif + } + } +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) + r = spadfs_setattr_common(dentry, iattr); +#else + r = spadfs_setattr_common(ns, dentry, iattr); +#endif + if (unlikely(r)) + return r; + + down_read_sync_lock(spadfnode(inode)->fs); + mutex_lock(&spadfnode(inode)->file_lock); + spadfs_update_ea(inode); + r = spadfs_write_file(spadfnode(inode), 0, NULL, NULL); + mutex_unlock(&spadfnode(inode)->file_lock); + up_read_sync_lock(spadfnode(inode)->fs); + + return r; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,23) || TEST_RHEL_VERSION(5,3) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) +static long spadfs_file_fallocate(struct inode *inode, int mode, loff_t offset, loff_t len) +{ +#else +static long spadfs_file_fallocate(struct file *file, int mode, loff_t offset, loff_t len) +{ + struct inode *inode = file->f_mapping->host; +#endif + SPADFS *fs = spadfnode(inode)->fs; + int r; + if (unlikely(mode & ~FALLOC_FL_KEEP_SIZE)) + return -EOPNOTSUPP; + inode_lock_nested(inode, 0); + offset += len; + r = 0; + if (!(mode & FALLOC_FL_KEEP_SIZE)) { + if (likely(offset > inode->i_size)) { + time_t t = ktime_get_real_seconds(); + inode_set_ctime(inode, t, 0); + inode->i_mtime.tv_sec = t; + inode->i_mtime.tv_nsec = 0; + set_target_size(spadfnode(inode), offset, 1); + r = generic_cont_expand_simple(inode, offset); + } + } else { + sector_t target_blocks = (unsigned long long)(offset + (512U << fs->sectors_per_disk_block_bits) - 1) >> (fs->sectors_per_disk_block_bits + 9); + sector_t existing_blocks = (unsigned long long)(spadfnode(inode)->disk_size + (512U << fs->sectors_per_disk_block_bits) - 1) >> (fs->sectors_per_disk_block_bits + 9); + if (likely(target_blocks > existing_blocks)) { + r = spadfs_extend_file(inode, target_blocks, 2); + } + } + inode_unlock(inode); + return r; +} +#endif + +#ifdef SPADFS_DIRECT_IO +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,16,0) +static ssize_t spadfs_direct_io(int rw, struct kiocb *iocb, + const struct iovec *iov, loff_t offset, + unsigned long nr_segs) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,1,0) +static ssize_t spadfs_direct_io(int rw, struct kiocb *iocb, + struct iov_iter *iter, loff_t offset) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) +static ssize_t spadfs_direct_io(struct kiocb *iocb, + struct iov_iter *iter, loff_t offset) +#else +static ssize_t spadfs_direct_io(struct kiocb *iocb, + struct iov_iter *iter) +#endif +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + int r; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0) + int rw = iov_iter_rw(iter); +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,16,0) + size_t count = iov_length(iov, nr_segs); +#else + size_t count = iov_iter_count(iter); +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,7,0) + loff_t offset = iocb->ki_pos; +#endif + + if (rw == WRITE) { + /* Copied from fat_direct_IO */ + loff_t size = offset + count; + if (spadfnode(inode)->mmu_private < size) + return 0; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0) + r = blockdev_direct_IO(rw, iocb, inode, inode->i_sb->s_bdev, iov, + offset, nr_segs, spadfs_get_block_direct, + NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3,16,0) + r = blockdev_direct_IO(rw, iocb, inode, iov, offset, nr_segs, + spadfs_get_block_direct); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,1,0) + r = blockdev_direct_IO(rw, iocb, inode, iter, offset, + spadfs_get_block_direct); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) + r = blockdev_direct_IO(iocb, inode, iter, offset, + spadfs_get_block_direct); +#else + r = blockdev_direct_IO(iocb, inode, iter, spadfs_get_block_direct); +#endif + + if (unlikely(r < 0) && rw == WRITE) { + spadfs_write_failed(file->f_mapping, offset + count); + } + + return r; +} +#endif + +#ifdef SPADFS_QUOTA + +static void *spadfs_read_file_sector(SPADFNODE *f, sector_t blk, struct buffer_head **bhp, int get_new) +{ + SPADFS *fs = f->fs; + struct buffer_head tmp_bh; + sector_t phys_sector; + int r; + void *file; + + tmp_bh.b_state = 0; + tmp_bh.b_size = 512U << fs->sectors_per_buffer_bits; + + r = spadfs_get_block(inode(f), blk, &tmp_bh, 0); + if (unlikely(r)) + return ERR_PTR(r); + + BUG_ON(!buffer_mapped(&tmp_bh)); + + phys_sector = tmp_bh.b_blocknr << fs->sectors_per_buffer_bits; + + if (likely(!get_new)) + file = spadfs_read_sector(fs, phys_sector, bhp, 0, + "spadfs_read_file_sector"); + else + file = spadfs_get_new_sector(fs, phys_sector, bhp, + "spadfs_read_file_sector"); + + return file; +} + +static noinline int spadfs_quota_extend(SPADFNODE *f, unsigned bytes) +{ + SPADFS *fs = f->fs; + loff_t i_size = i_size_read(inode(f)); + int r; + + if (i_size >= f->disk_size) { + BUG_ON(i_size != f->disk_size); + + r = spadfs_do_extend(f, i_size + (512U << fs->sectors_per_buffer_bits), 0, 0); + if (unlikely(r)) + return r; + } + + i_size += bytes; + BUG_ON(i_size > f->disk_size); + i_size_write(inode(f), i_size); + f->mmu_private = i_size; + + spadfs_write_file(f, 0, NULL, NULL); + + return 0; +} + +static ssize_t spadfs_quota_rw(struct super_block *s, struct inode *inode, + char *data, size_t len, loff_t position, int rw) +{ + SPADFS *fs = spadfs(s); + size_t bytes_left; + int r; + + if (unlikely(position > i_size_read(inode))) + return 0; + + bytes_left = len; + while (bytes_left > 0) { + unsigned off; + unsigned to_copy; + struct buffer_head *bh; + char *file; + loff_t i_size; + + i_size = i_size_read(inode); + + off = position & ((512U << fs->sectors_per_buffer_bits) - 1); + to_copy = (512U << fs->sectors_per_buffer_bits) - off; + if (to_copy > bytes_left) + to_copy = bytes_left; + + if (unlikely(position == i_size)) { + if (!rw) + break; + r = spadfs_quota_extend(spadfnode(inode), to_copy); + if (unlikely(r)) + return r; + i_size = i_size_read(inode); + } + + if (unlikely(position + to_copy > i_size)) + to_copy = i_size - position; + + file = spadfs_read_file_sector(spadfnode(inode), + position >> (fs->sectors_per_buffer_bits + 9), &bh, + rw && to_copy == 512U << fs->sectors_per_buffer_bits); + if (unlikely(IS_ERR(file))) + return PTR_ERR(file); + + if (!rw) + memcpy(data, file + off, to_copy); + else { + lock_buffer(bh); + memcpy(file + off, data, to_copy); + flush_dcache_page(bh->b_page); + unlock_buffer(bh); + mark_buffer_dirty(bh); + } + + spadfs_brelse(fs, bh); + + data += to_copy; + position += to_copy; + bytes_left -= to_copy; + } + return len - bytes_left; +} + +ssize_t spadfs_quota_read(struct super_block *s, int type, + char *data, size_t len, loff_t position) +{ + struct inode *inode = sb_dqopt(s)->files[type]; + + return spadfs_quota_rw(s, inode, data, len, position, 0); +} + +ssize_t spadfs_quota_write(struct super_block *s, int type, + const char *data, size_t len, loff_t position) +{ + struct inode *inode = sb_dqopt(s)->files[type]; + SPADFS *fs = spadfs(s); + ssize_t r; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) + inode_lock_nested(inode, I_MUTEX_QUOTA); +#endif + mutex_lock(&fs->quota_alloc_lock); + + r = spadfs_quota_rw(s, inode, (char *)data, len, position, 1); + + mutex_unlock(&fs->quota_alloc_lock); +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) + inode_unlock(inode); +#endif + + return r; +} + +#if defined(SPADFS_QUOTA) && SPADFS_QUOTA >= 2 +struct dquot **spadfs_quota_get(struct inode *inode) +{ + return spadfnode(inode)->i_dquot; +} +#endif + +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21) +const +#endif +struct inode_operations spadfs_file_iops = { +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,8,0) + .truncate = spadfs_truncate, +#endif + .setattr = spadfs_file_setattr, + .getattr = spadfs_getattr, +#ifdef SPADFS_XATTR +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,9,0) + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .removexattr = generic_removexattr, +#endif + .listxattr = spadfs_listxattr, +#endif +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,23) && LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38)) || TEST_RHEL_VERSION(5,3) + .fallocate = spadfs_file_fallocate, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,28) || TEST_RHEL_VERSION(5,4) + .fiemap = spadfs_fiemap, +#endif +}; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17) +const +#endif +struct file_operations spadfs_file_fops = { + .llseek = generic_file_llseek, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,16,0) + .read = do_sync_read, +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,1,0) + .read = new_sync_read, +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,1,0) + .write = spadfs_file_write, +#endif + .mmap = generic_file_mmap, + .fsync = spadfs_file_fsync, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,16,0) + .aio_read = generic_file_aio_read, + .aio_write = generic_file_aio_write, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17) + .splice_read = generic_file_splice_read, + .splice_write = generic_file_splice_write, +#endif +#else + .read_iter = generic_file_read_iter, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,1,0) + .write_iter = generic_file_write_iter, +#else + .write_iter = spadfs_file_write_iter, +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,5,0) + .splice_read = generic_file_splice_read, +#else + .splice_read = filemap_splice_read, +#endif + .splice_write = iter_file_splice_write, +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) + .sendfile = generic_file_sendfile, +#endif + .release = spadfs_release, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,11) + .unlocked_ioctl = spadfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = spadfs_compat_ioctl, +#endif +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,38) + .fallocate = spadfs_file_fallocate, +#endif +}; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) +const +#endif +struct address_space_operations spadfs_file_aops = { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,18,0) + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(5,14,0) + .set_page_dirty = __set_page_dirty_buffers, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0) + .read_folio = spadfs_read_folio, +#else + .readpage = spadfs_readpage, +#endif + .writepage = spadfs_writepage, +#ifdef SPADFS_MPAGE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,8,0) || TEST_RHEL_VERSION(8,7) + .readahead = spadfs_readahead, +#else + .readpages = spadfs_readpages, +#endif + .writepages = spadfs_writepages, +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,39) + .sync_page = block_sync_page, +#endif +#ifdef SPADFS_DIRECT_IO + .direct_IO = spadfs_direct_io, +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + .prepare_write = spadfs_prepare_write, +#else + .write_begin = spadfs_write_begin, + .write_end = spadfs_write_end, +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) + .commit_write = generic_commit_write, +#endif + .bmap = spadfs_bmap, +}; + diff --git a/fs/spadfs/inode.c b/fs/spadfs/inode.c new file mode 100644 index 000000000..4d6a77b15 --- /dev/null +++ b/fs/spadfs/inode.c @@ -0,0 +1,814 @@ +#include "spadfs.h" + +/* Helpers for iget5_locked */ + +static int spadfs_iget_test(struct inode *inode, void *data) +{ + spadfs_ino_t *ino = data; + return spadfnode(inode)->spadfs_ino == *ino; +} + +static int spadfs_iget_init(struct inode *inode, void *data) +{ + spadfs_ino_t *ino = data; +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37) + static atomic_t next_ino = ATOMIC_INIT(0); + inode->i_ino = (unsigned)atomic_inc_return(&next_ino) - 1; +#else + inode->i_ino = get_next_ino(); +#endif + spadfnode(inode)->spadfs_ino = *ino; + spadfnode(inode)->stable_ino = 0; + return 0; +} + +struct fnode_ea *spadfs_get_ea(SPADFNODE *f, u32 what, u32 mask) +{ + struct fnode_ea *ea = GET_EA((struct fnode_ea *)f->ea, f->ea_size, + what, mask); + if (unlikely(ea == GET_EA_ERROR)) { + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "error parsing extended attributes on fnode %Lx/%x", + (unsigned long long)f->fnode_block, + f->fnode_pos); + return ERR_PTR(-EFSERROR); + } + return ea; +} + +void spadfs_find_ea_unx(SPADFNODE *f) +{ + struct ea_unx *ea; + ea = (struct ea_unx *)spadfs_get_ea(f, EA_UNX_MAGIC, EA_UNX_MAGIC_MASK); + if (unlikely(IS_ERR(ea))) + ea = NULL; + f->ea_unx = ea; +} + +void spadfs_validate_stable_ino(u64 *result, u64 ino) +{ + if (likely(ino > SPADFS_INO_ROOT)) + *result = ino; +} + +u64 spadfs_expose_inode_number(SPADFS *fs, u64 ino) +{ + if (unlikely(fs->mount_flags & MOUNT_FLAGS_64BIT_INO_FORCE)) { + if (likely(!(ino & 0xFFFFFFFF00000000ULL))) + ino |= 0xFFFFFFFF00000000ULL; + } + if (!(fs->mount_flags & MOUNT_FLAGS_64BIT_INO)) + ino = (u32)ino; + if (unlikely(ino < SPADFS_INO_ROOT)) + ino += SPADFS_INO_INITIAL_REGION; + return ino; +} + +/* + * Read extended attributes and fill inode bits. On error, do not return error + * and continue as if there were no extended attributes. However, error is + * written to syslog and TXFLAGS_EA_ERROR bit is set on filesystem, so that fsck + * will automatically correct it on next reboot. + */ + +static void spadfs_get_ea_unx(struct inode *inode, struct fnode *fnode) +{ + umode_t mode; + struct ea_unx *ea; + struct fnode_ea *lnk; + SPADFS *fs = spadfnode(inode)->fs; + lnk = spadfs_get_ea(spadfnode(inode), + EA_SYMLINK_MAGIC, EA_SYMLINK_MAGIC_MASK); + if (unlikely(IS_ERR(lnk))) + return; + if (unlikely(lnk != NULL) && likely(!inode->i_size)) { + inode->i_mode = S_IFLNK | S_IRWXUGO; + inode->i_op = &spadfs_symlink_iops; + inode->i_size = (SPAD2CPU32_LV(&lnk->magic) >> FNODE_EA_SIZE_SHIFT) & + FNODE_EA_SIZE_MASK_1; + if (fnode) { + spadfs_validate_stable_ino(&spadfnode(inode)->stable_ino, + ((u64)SPAD2CPU32_LV(&fnode->run10)) | + ((u64)SPAD2CPU16_LV(&fnode->run11) << 32) | + ((u64)SPAD2CPU16_LV(&fnode->run1n) << 48)); + i_uid_write(inode, SPAD2CPU32_LV(&fnode->run20)); + i_gid_write(inode, SPAD2CPU16_LV(&fnode->run21) | ((u32)SPAD2CPU16_LV(&fnode->run2n) << 16)); + } + return; + } + + spadfs_find_ea_unx(spadfnode(inode)); + ea = spadfnode(inode)->ea_unx; + if (unlikely(!ea)) + return; + if (likely(SPAD2CPU32_LV(&ea->magic) == EA_UNX_MAGIC)) + spadfs_validate_stable_ino(&spadfnode(inode)->stable_ino, + SPAD2CPU64_LV(&ea->ino)); + mode = SPAD2CPU16_LV(&ea->mode); + if (S_ISDIR(mode)) { + if (unlikely(!S_ISDIR(inode->i_mode))) { + spadfs_error(fs, TXFLAGS_EA_ERROR, + "UNX extended attribute error on fnode %Lx/%x: non-directory has directory mode %06o", + (unsigned long long)spadfnode(inode)->fnode_block, + spadfnode(inode)->fnode_pos, mode); + return; + } + } else { + if (unlikely(!S_ISREG(inode->i_mode))) { + spadfs_error(fs, TXFLAGS_EA_ERROR, + "UNX extended attribute error on fnode %Lx/%x: directory has non-directory mode %06o", + (unsigned long long)spadfnode(inode)->fnode_block, + spadfnode(inode)->fnode_pos, + mode); + return; + } + if (unlikely(!S_ISREG(mode))) { + dev_t rdev = 0; + if (likely(S_ISCHR(mode)) || S_ISBLK(mode)) { + struct ea_rdev *rd = (struct ea_rdev *) + spadfs_get_ea(spadfnode(inode), + EA_RDEV_MAGIC, ~0); + if (unlikely(IS_ERR(rd))) + return; + if (unlikely(!rd)) { + spadfs_error(fs, TXFLAGS_EA_ERROR, + "UNX extended attribute error on fnode %Lx/%x: no rdev attribute for fnode with mode %06o", + (unsigned long long)spadfnode(inode)->fnode_block, + spadfnode(inode)->fnode_pos, + mode); + return; + } + rdev = huge_decode_dev(SPAD2CPU64_LV(&rd->dev)); + } else if (S_ISFIFO(mode) || likely(S_ISSOCK(mode))) { + } else { + spadfs_error(fs, TXFLAGS_EA_ERROR, + "UNX extended attribute error on fnode %Lx/%x: file has invalid mode %06o", + (unsigned long long)spadfnode(inode)->fnode_block, + spadfnode(inode)->fnode_pos, + mode); + return; + } + init_special_inode(inode, mode, rdev); + } + } + if (likely(fnode != NULL)) { + if (likely(S_ISREG(mode))) { + unsigned prealloc = SPAD2CPU32_LV(&ea->prealloc[ + !CC_VALID(fs, &fnode->cc, &fnode->txc)]); + if (unlikely(prealloc > inode->i_size)) { + spadfs_error(fs, TXFLAGS_EA_ERROR, + "UNX extended attribute error on fnode %Lx/%x: prealloc (%d) is larger than file size (%Ld)", + (unsigned long long)spadfnode(inode)->fnode_block, + spadfnode(inode)->fnode_pos, + prealloc, + (unsigned long long)inode->i_size); + return; + } + inode->i_size -= prealloc; + spadfnode(inode)->mmu_private = inode->i_size; + } else { + if (unlikely((SPAD2CPU32_LV(&ea->prealloc[0]) | + SPAD2CPU32_LV(&ea->prealloc[1])) != 0)) { + spadfs_error(fs, TXFLAGS_EA_ERROR, + "UNX extended attribute error on fnode %Lx/%x: fnode with mode %06o has non-zero prealloc", + (unsigned long long)spadfnode(inode)->fnode_block, + spadfnode(inode)->fnode_pos, + mode); + return; + } + } + } + inode->i_mode = mode; + i_uid_write(inode, SPAD2CPU32_LV(&ea->uid)); + i_gid_write(inode, SPAD2CPU32_LV(&ea->gid)); +} + +int spadfs_get_fixed_fnode_pos(SPADFS *fs, struct fnode_block *fnode_block, + sector_t secno, unsigned *pos) +{ +#define fx ((struct fixed_fnode_block *)fnode_block) + *pos = FIXED_FNODE_BLOCK_FNODE1 - CC_VALID(fs, &fx->cc, &fx->txc) * + (FIXED_FNODE_BLOCK_FNODE1 - FIXED_FNODE_BLOCK_FNODE0); + if (unlikely(*(u64 *)((char *)fnode_block + *pos - + (FIXED_FNODE_BLOCK_FNODE0 - FIXED_FNODE_BLOCK_NLINK0)) == + CPU2SPAD64_CONST(0))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "fixed fnode %Lx has zero nlink", + (unsigned long long)secno); + return -EFSERROR; + } + return 0; +#undef fx +} + +/* + * Read an inode. The inode to read is determined by (already filled) spadfs_ino + * entry. The sector and position of inode is determined by spadfs_ino_t_sec and + * spadfs_ino_t_pos macros from spadfs_ino. If position is 0, it is fixed fnode + * (i.e. hardlink was created before to it) --- in this case spadfs_ino_t_sec + * points to fixed fnode block. + * If the fnode is in directory, parent is inode number of that directory, if + * the fnode is fixed, parent is "don't care". + */ + +static int spadfs_read_inode(struct inode *inode, sector_t parent_block, unsigned parent_pos) +{ + int r; + struct fnode *fnode; + struct buffer_head *bh; + unsigned ea_pos, ea_size; + int cc_valid; + loff_t other_i_size; + SPADFS *fs = spadfs(inode->i_sb); + unsigned pos = spadfs_ino_t_pos(spadfnode(inode)->spadfs_ino); + struct fnode_block *fnode_block; + + spadfnode(inode)->fnode_block = + spadfs_ino_t_sec(spadfnode(inode)->spadfs_ino); + fnode_block = spadfs_read_fnode_block(fs, + spadfnode(inode)->fnode_block, + &bh, + SRFB_FNODE + unlikely(is_pos_fixed(pos)) * (SRFB_FIXED_FNODE - SRFB_FNODE), + "spadfs_read_inode"); + if (unlikely(IS_ERR(fnode_block))) { + r = PTR_ERR(fnode_block); + goto make_bad; + } + + if (unlikely(is_pos_fixed(pos))) { + parent_block = 0; + parent_pos = 0; + r = spadfs_get_fixed_fnode_pos(fs, fnode_block, + spadfnode(inode)->fnode_block, + &pos); + if (unlikely(r)) + goto brelse_make_bad; + } + spadfnode(inode)->fnode_pos = pos; + fnode = (struct fnode *)((char *)fnode_block + pos); + + ea_pos = FNODE_EA_POS(fnode->namelen); + ea_size = (SPAD2CPU16_LV(&fnode->next) & FNODE_NEXT_SIZE) - ea_pos; + if (unlikely(pos + ea_pos + ea_size > FNODE_BLOCK_SIZE) || + unlikely(ea_size > FNODE_MAX_EA_SIZE)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "invalid extended attributes on fnode %Lx/%x", + (unsigned long long)spadfnode(inode)->fnode_block, + spadfnode(inode)->fnode_pos); + r = -EFSERROR; + goto brelse_make_bad; + } + if (unlikely(fnode->flags & FNODE_FLAGS_HARDLINK)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "attempting to read hardlink as fnode: %Lx/%x", + (unsigned long long)spadfnode(inode)->fnode_block, + spadfnode(inode)->fnode_pos); + r = -EFSERROR; + goto brelse_make_bad; + } + if (unlikely(r = spadfs_ea_resize(spadfnode(inode), ea_size))) + goto brelse_make_bad; + spadfnode(inode)->ea_size = ea_size; + memcpy(spadfnode(inode)->ea, (char *)fnode + ea_pos, ea_size); + inode->i_mode = fs->mode; + i_uid_write(inode, fs->uid); + i_gid_write(inode, fs->gid); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,18,0) + inode->i_atime = current_kernel_time(); +#else + ktime_get_coarse_real_ts64(&inode->i_atime); +#endif + inode_set_ctime(inode, SPAD2CPU32_LV(&fnode->ctime), 0); + inode->i_mtime.tv_sec = SPAD2CPU32_LV(&fnode->mtime); + inode->i_mtime.tv_nsec = 0; + inode->i_generation = 0; + spadfs_set_parent_fnode(spadfnode(inode), parent_block, parent_pos); + spadfnode(inode)->ea_unx = NULL; + spadfnode(inode)->spadfs_nlink = 1; + spadfnode(inode)->commit_sequence = + CC_CURRENT(fs, &fnode->cc, &fnode->txc) ? + fs->commit_sequence : 0; + cc_valid = CC_VALID(fs, &fnode->cc, &fnode->txc); + inode->i_size = SPAD2CPU64_LV(&fnode->size[!cc_valid]); + other_i_size = SPAD2CPU64_LV(&fnode->size[cc_valid]); + if (likely(!(fnode->flags & FNODE_FLAGS_DIR))) { + unsigned i; + if (unlikely(is_fnode_fixed(spadfnode(inode)))) + spadfnode(inode)->spadfs_nlink = SPAD2CPU64_LV(FIXED_FNODE_NLINK_PTR(fnode)); + + inode->i_mode = (inode->i_mode & ~0111) | S_IFREG; + spadfnode(inode)->mmu_private = inode->i_size; + spadfnode(inode)->disk_size = spadfs_roundup_blocksize(fs, inode->i_size); + spadfnode(inode)->crash_disk_size = spadfs_roundup_blocksize(fs, other_i_size); + inode->i_blocks = spadfs_size_2_sectors(fs, spadfnode(inode)->disk_size); + + spadfnode(inode)->blk1 = MAKE_D_OFF(fnode->run10, fnode->run11); + spadfnode(inode)->blk1_n = SPAD2CPU16_LV(&fnode->run1n); + spadfnode(inode)->blk2 = MAKE_D_OFF(fnode->run20, fnode->run21); + spadfnode(inode)->blk2_n = SPAD2CPU16_LV(&fnode->run2n); + + for (i = 0; i < spadfs_extent_cache_size; i++) { + spadfnode(inode)->extent_cache[i].physical_sector = spadfnode(inode)->blk1; + spadfnode(inode)->extent_cache[i].logical_sector = 0; + spadfnode(inode)->extent_cache[i].n_sectors = spadfnode(inode)->blk1_n; + } + + spadfnode(inode)->root = MAKE_D_OFF(fnode->anode0, fnode->anode1); + inode->i_op = &spadfs_file_iops; + inode->i_fop = &spadfs_file_fops; + inode->i_data.a_ops = &spadfs_file_aops; + } else { + if (unlikely(!(parent_block | parent_pos))) + spadfnode(inode)->stable_ino = SPADFS_INO_ROOT; + inode->i_mode |= S_IFDIR; + inode->i_blocks = inode->i_size >> 9; + spadfnode(inode)->root = cc_valid ? + MAKE_D_OFF(fnode->run10, fnode->run11) : + MAKE_D_OFF(fnode->run20, fnode->run21); + inode->i_op = &spadfs_dir_iops; + inode->i_fop = &spadfs_dir_fops; + } + spadfs_get_ea_unx(inode, fnode); +/* See the comment about internal and external nlink counts at spadfs_set_nlink */ + spadfs_set_nlink(inode); + spadfs_brelse(fs, bh); + return 0; + +brelse_make_bad: + spadfs_brelse(fs, bh); +make_bad: + make_bad_inode(inode); + return r; +} + +/* Get an inode from cache or read it from disk */ + +struct inode *spadfs_iget(struct super_block *s, spadfs_ino_t spadfs_ino, + sector_t parent_block, unsigned parent_pos) +{ + struct inode *inode = iget5_locked(s, spadfs_ino_t_2_ino_t(spadfs_ino), + spadfs_iget_test, spadfs_iget_init, &spadfs_ino); + if (unlikely(!inode)) { + printk(KERN_ERR "spadfs: unable to allocate inode\n"); + return ERR_PTR(-ENOMEM); + } + + if (unlikely(inode->i_state & I_NEW)) { + int r = spadfs_read_inode(inode, parent_block, parent_pos); + unlock_new_inode(inode); + if (unlikely(r)) { + iput(inode); + return ERR_PTR(r); + } + } + + return inode; +} + +struct inode *spadfs_new_inode(struct inode *dir, umode_t mode, time_t ctime, + void *ea, unsigned ea_size, + sector_t fnode_block, unsigned fnode_pos, + sector_t dir_root, u64 stable_ino) +{ + SPADFS *fs = spadfnode(dir)->fs; + spadfs_ino_t ino = make_spadfs_ino_t(fnode_block, fnode_pos); + struct inode *inode; + int r; + + inode = new_inode(dir->i_sb); + if (unlikely(!inode)) { + r = -ENOMEM; + goto ret_r; + } + + spadfs_iget_init(inode, &ino); + i_uid_write(inode, 0); + i_gid_write(inode, 0); + inode->i_mode = mode; + inode->i_blocks = 0; + inode->i_generation = 0; + inode_set_ctime(inode, ctime, 0); + inode->i_mtime.tv_sec = ctime; + inode->i_mtime.tv_nsec = 0; + inode->i_atime.tv_sec = ctime; + inode->i_atime.tv_nsec = 0; + set_nlink(inode, 1); + spadfnode(inode)->spadfs_nlink = 1; + spadfnode(inode)->disk_size = 0; + spadfnode(inode)->crash_disk_size = 0; + spadfnode(inode)->commit_sequence = fs->commit_sequence; + spadfnode(inode)->mmu_private = 0; + spadfs_validate_stable_ino(&spadfnode(inode)->stable_ino, stable_ino); + spadfnode(inode)->blk1 = 0; + spadfnode(inode)->blk2 = 0; + spadfnode(inode)->blk1_n = 0; + spadfnode(inode)->blk2_n = 0; + memset(&spadfnode(inode)->extent_cache, 0, + sizeof spadfnode(inode)->extent_cache); + spadfnode(inode)->root = dir_root; + spadfnode(inode)->fnode_block = fnode_block; + spadfnode(inode)->fnode_pos = fnode_pos; + spadfs_set_parent_fnode(spadfnode(inode), spadfnode(dir)->fnode_block, spadfnode(dir)->fnode_pos); + spadfnode(inode)->ea_unx = NULL; + if (unlikely(r = spadfs_ea_resize(spadfnode(inode), ea_size))) + goto drop_inode_ret_r; + spadfnode(inode)->ea_size = ea_size; + memcpy(spadfnode(inode)->ea, ea, ea_size); + BUG_ON(ea_size & (FNODE_EA_ALIGN - 1)); + if (unlikely(S_ISDIR(mode))) { + inode->i_op = &spadfs_dir_iops; + inode->i_fop = &spadfs_dir_fops; + inode->i_size = directory_size(fs) ? 512U << fs->sectors_per_disk_block_bits : 0; + } else { + inode->i_op = &spadfs_file_iops; + inode->i_fop = &spadfs_file_fops; + inode->i_data.a_ops = &spadfs_file_aops; + } + spadfs_get_ea_unx(inode, NULL); + +#ifdef SPADFS_QUOTA + dquot_initialize(inode); + r = dquot_alloc_inode(inode); + if (unlikely(r)) + goto drop_inode_ret_r; +#endif + + if (unlikely(S_ISDIR(mode)) && + likely(directory_size(fs))) { +#ifdef SPADFS_QUOTA + r = dquot_alloc_space_nodirty(inode, inode->i_size); + if (unlikely(r)) + goto free_iquota_drop_inode_ret_r; +#else + inode_add_bytes(inode, inode->i_size); +#endif + } + + __insert_inode_hash(inode, spadfs_ino_t_2_ino_t(spadfnode(inode)->spadfs_ino)); + return inode; + +#ifdef SPADFS_QUOTA +free_iquota_drop_inode_ret_r: + dquot_free_inode(inode); +#endif + +drop_inode_ret_r: +#ifdef SPADFS_QUOTA + dquot_drop(inode); +#endif + inode->i_flags |= S_NOQUOTA; + clear_nlink(inode); + iput(inode); + +ret_r: + return ERR_PTR(r); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) + +void spadfs_delete_inode(struct inode *i) +{ + sync_lock_decl + SPADFS *fs; + + BUG_ON(spadfnode(i)->res.len); + BUG_ON(!list_empty(&spadfnode(i)->clear_entry)); + + if (unlikely(S_ISDIR(i->i_mode))) goto clr; + truncate_inode_pages(&i->i_data, 0); + fs = spadfnode(i)->fs; + down_read_sync_lock(fs); + /* no one can modify this inode anyway */ + /*mutex_lock(&spadfnode(i)->file_lock);*/ + spadfs_delete_file_content(spadfnode(i)); + /*mutex_unlock(&spadfnode(i)->file_lock);*/ + up_read_sync_lock(fs); +clr: + clear_inode(i); +} + +#else + +void spadfs_evict_inode(struct inode *i) +{ + sync_lock_decl + SPADFS *fs; + int want_delete = 0; + + BUG_ON(spadfnode(i)->res.len); + BUG_ON(!list_empty(&spadfnode(i)->clear_entry)); + + if (!i->i_nlink && !is_bad_inode(i)) { + want_delete = 1; +#ifdef SPADFS_QUOTA + dquot_initialize(i); + dquot_free_inode(i); +#endif + } + + truncate_inode_pages(&i->i_data, 0); + + if (!want_delete || unlikely(S_ISDIR(i->i_mode))) + goto end; + + fs = spadfnode(i)->fs; + down_read_sync_lock(fs); + /* no one can modify this inode anyway */ + /*mutex_lock(&spadfnode(i)->file_lock);*/ + spadfs_delete_file_content(spadfnode(i)); + /*mutex_unlock(&spadfnode(i)->file_lock);*/ + + up_read_sync_lock(fs); + +end: +#ifdef SPADFS_QUOTA + dquot_drop(i); +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) + end_writeback(i); +#else + clear_inode(i); +#endif +} + +#endif + +u64 spadfs_get_user_inode_number(struct inode *inode) +{ + u64 result; + + /* + * Kernel returns -EOVERFLOW on stat32 if ino >= 2^32. + * This would cause random program fails. + */ + + if (likely(spadfnode(inode)->stable_ino != 0)) + result = spadfnode(inode)->stable_ino; + else + result = spadfs_ino_t_2_ino64_t(spadfnode(inode)->spadfs_ino); + + return spadfs_expose_inode_number(spadfnode(inode)->fs, result); +} + +/* Initial UNX attribute */ + +void spadfs_get_initial_attributes(struct inode *dir, umode_t *mode, uid_t *uid, gid_t *gid) +{ + *uid = get_current_fsuid(); + *gid = get_current_fsgid(); + if (unlikely(dir->i_mode & S_ISGID)) { + *gid = i_gid_read(dir); + if (unlikely(S_ISDIR(*mode))) { + *mode |= S_ISGID; + } + } +} + +void spadfs_init_unx_attribute(struct inode *dir, struct ea_unx *ea, + umode_t mode, u64 stable_ino) +{ + uid_t uid; + gid_t gid; + + CPU2SPAD32_LV(&ea->magic, EA_UNX_MAGIC); + CPU2SPAD16_LV(&ea->flags, 0); + CPU2SPAD32_LV(&ea->prealloc[0], 0); + CPU2SPAD32_LV(&ea->prealloc[1], 0); + CPU2SPAD64_LV(&ea->ino, stable_ino); + + spadfs_get_initial_attributes(dir, &mode, &uid, &gid); + + CPU2SPAD32_LV(&ea->uid, uid); + CPU2SPAD32_LV(&ea->gid, gid); + CPU2SPAD16_LV(&ea->mode, mode); +} + +static noinline int spadfs_make_unx_attribute(struct dentry *dentry) +{ +#define new_ea_size FNODE_EA_DO_ALIGN(sizeof(struct ea_unx) - 8) + int r; + SPADFNODE *f = spadfnode(dentry->d_inode); + SPADFS *fs = f->fs; + u8 *ea; + unsigned ea_size; + struct ea_unx *ea_unx; + /*sync_lock_decl*/ + + ea = kmalloc(FNODE_MAX_EA_SIZE, GFP_NOIO); + if (unlikely(!ea)) + return -ENOMEM; + + down_write_sync_lock(fs); + + if (unlikely(f->ea_unx != NULL)) { + r = 0; + goto unlock_ret_r; + } + + ea_size = f->ea_size; + if (unlikely(ea_size + new_ea_size > FNODE_MAX_EA_SIZE)) { + r = -EOVERFLOW; + goto unlock_ret_r; + } + + memcpy(ea, f->ea, ea_size); + ea_unx = (struct ea_unx *)(ea + ea_size); + CPU2SPAD32_LV(&ea_unx->magic, EA_UNX_MAGIC_OLD); + CPU2SPAD16_LV(&ea_unx->flags, 0); + CPU2SPAD32_LV(&ea_unx->prealloc[0], 0); + CPU2SPAD32_LV(&ea_unx->prealloc[1], 0); + CPU2SPAD32_LV(&ea_unx->uid, i_uid_read(&f->vfs_inode)); + CPU2SPAD32_LV(&ea_unx->gid, i_gid_read(&f->vfs_inode)); + CPU2SPAD16_LV(&ea_unx->mode, f->vfs_inode.i_mode); + ea_size += new_ea_size; + + r = spadfs_refile_fnode(spadfnode(dentry->d_parent->d_inode), &dentry->d_name, f, ea, ea_size); + +unlock_ret_r: + up_write_sync_lock(fs); + kfree(ea); + + return r; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) +int spadfs_setattr_common(struct dentry *dentry, struct iattr *iattr) +#else +int spadfs_setattr_common(struct mnt_idmap *ns, struct dentry *dentry, struct iattr *iattr) +#endif +{ + struct inode *inode = dentry->d_inode; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,9,0) && !TEST_STABLE_BRANCH(3,2,84) && !TEST_STABLE_BRANCH(3,16,39) && !TEST_STABLE_BRANCH(4,1,37) + int r = inode_change_ok(inode, iattr); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) + int r = setattr_prepare(dentry, iattr); +#else + int r = setattr_prepare(ns, dentry, iattr); +#endif + if (unlikely(r)) + return r; + + if (likely(!S_ISLNK(inode->i_mode))) { + if (unlikely(!spadfnode(inode)->ea_unx) && + iattr->ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID)) { + r = spadfs_make_unx_attribute(dentry); + if (unlikely(r)) + return r; + } + } + +#ifdef SPADFS_QUOTA +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,0,0) + if (is_quota_modification(inode, iattr)) + dquot_initialize(inode); + if ((iattr->ia_valid & ATTR_UID && !uid_eq(iattr->ia_uid, inode->i_uid)) || + (iattr->ia_valid & ATTR_GID && !gid_eq(iattr->ia_gid, inode->i_gid))) { + r = dquot_transfer(inode, iattr); + if (unlikely(r)) + return r; + } +#else + if (is_quota_modification(ns, inode, iattr)) + dquot_initialize(inode); + if (i_uid_needs_update(ns, iattr, inode) || + i_gid_needs_update(ns, iattr, inode)) { + r = dquot_transfer(ns, inode, iattr); + if (unlikely(r)) + return r; + } +#endif +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) + return inode_setattr(inode, iattr); +#else + if (iattr->ia_valid & ATTR_SIZE) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,8,0) + r = vmtruncate(inode, iattr->ia_size); + if (unlikely(r)) + return r; +#else + truncate_setsize(inode, iattr->ia_size); + spadfs_truncate(inode); +#endif + } +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) + setattr_copy(inode, iattr); +#else + setattr_copy(ns, inode, iattr); +#endif + return 0; +#endif +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0) +int spadfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) +int spadfs_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned query_flags) +#else +int spadfs_getattr(struct mnt_idmap *ns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned query_flags) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0) + struct inode *inode = dentry->d_inode; + generic_fillattr(inode, stat); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) + struct inode *inode = d_inode(path->dentry); + generic_fillattr(inode, stat); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(6,6,0) + struct inode *inode = d_inode(path->dentry); + generic_fillattr(ns, inode, stat); +#else + struct inode *inode = d_inode(path->dentry); + generic_fillattr(ns, request_mask, inode, stat); +#endif + stat->blksize = spadfnode(inode)->fs->xfer_size; + stat->ino = spadfs_get_user_inode_number(inode); + return 0; +} + +void spadfs_update_ea(struct inode *inode) +{ + if (unlikely(!spadfnode(inode)->ea_unx)) + return; + CPU2SPAD16_LV(&spadfnode(inode)->ea_unx->mode, inode->i_mode); + CPU2SPAD32_LV(&spadfnode(inode)->ea_unx->uid, i_uid_read(inode)); + CPU2SPAD32_LV(&spadfnode(inode)->ea_unx->gid, i_gid_read(inode)); +} + +static unsigned spadfs_parent_hash(sector_t sec, unsigned pos) +{ + unsigned n; + n = ((unsigned long)sec / SPADFS_INODE_HASH_SIZE) ^ + (unsigned long)sec ^ + ((unsigned long)pos * (SPADFS_INODE_HASH_SIZE / 512)); + return n & (SPADFS_INODE_HASH_SIZE - 1); +} + +static void spadfs_set_parent_fnode_unlocked(SPADFNODE *f, sector_t sec, unsigned pos) +{ + SPADFS *fs = f->fs; + if (f->parent_fnode_block) { + hlist_del(&f->inode_list); + } + f->parent_fnode_block = sec; + f->parent_fnode_pos = pos; + if (sec) { + hlist_add_head(&f->inode_list, &fs->inode_list[spadfs_parent_hash(sec, pos)]); + } +} + +void spadfs_set_parent_fnode(SPADFNODE *f, sector_t sec, unsigned pos) +{ + mutex_lock(&f->fs->inode_list_lock); + spadfs_set_parent_fnode_unlocked(f, sec, pos); + mutex_unlock(&f->fs->inode_list_lock); +} + +void spadfs_move_parent_dir_ptr(SPADFS *fs, sector_t src_sec, unsigned src_pos, + sector_t dst_sec, unsigned dst_pos) +{ + SPADFNODE *f; + unsigned h; + unsigned char s = 0; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) + struct hlist_node *hn; +#endif + struct hlist_node *n; + + mutex_lock(&fs->inode_list_lock); + h = spadfs_parent_hash(src_sec, src_pos); +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) + hlist_for_each_entry_safe(f, hn, n, &fs->inode_list[h], inode_list) +#else + hlist_for_each_entry_safe(f, n, &fs->inode_list[h], inode_list) +#endif + { + if (f->parent_fnode_block == src_sec && + f->parent_fnode_pos == src_pos) { + spadfs_set_parent_fnode_unlocked(f, dst_sec, dst_pos); + } + if (!++s) + spadfs_cond_resched(); + } + mutex_unlock(&fs->inode_list_lock); +} + +void spadfs_move_fnode_ptr(SPADFS *fs, sector_t src_sec, unsigned src_pos, + sector_t dst_sec, unsigned dst_pos, int is_dir) +{ + spadfs_ino_t spadfs_ino = make_spadfs_ino_t(src_sec, src_pos); + struct inode *inode = ilookup5(fs->s, spadfs_ino_t_2_ino_t(spadfs_ino), + spadfs_iget_test, &spadfs_ino); + if (unlikely(inode != NULL)) { + spadfnode(inode)->fnode_block = dst_sec; + spadfnode(inode)->fnode_pos = dst_pos; + spadfnode(inode)->spadfs_ino = make_spadfs_ino_t(dst_sec, dst_pos); + remove_inode_hash(inode); + __insert_inode_hash(inode, spadfs_ino_t_2_ino_t(spadfnode(inode)->spadfs_ino)); + iput(inode); + } + if (unlikely(is_dir)) + spadfs_move_parent_dir_ptr(fs, src_sec, src_pos, + dst_sec, dst_pos); +} + diff --git a/fs/spadfs/ioctl.c b/fs/spadfs/ioctl.c new file mode 100644 index 000000000..e10d5ee56 --- /dev/null +++ b/fs/spadfs/ioctl.c @@ -0,0 +1,37 @@ +#include "spadfs.h" + +long spadfs_ioctl(struct file *file, unsigned cmd, unsigned long arg) +{ + switch (cmd) { +#ifdef SPADFS_FSTRIM + case FITRIM: { + SPADFS *fs = spadfs(file_inode(file)->i_sb); + struct fstrim_range range; + sector_t n_trimmed; + int r; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&range, (struct fstrim_range __user *)arg, sizeof(range))) + return -EFAULT; + r = spadfs_trim_fs(fs, range.start >> 9, (range.start + range.len) >> 9, (range.minlen + 511) >> 9, &n_trimmed); + if (r) + return r; + range.len = (u64)n_trimmed << 9; + if (copy_to_user((struct fstrim_range __user *)arg, &range, sizeof(range))) + return -EFAULT; + return 0; + } +#endif + default: { + return -ENOIOCTLCMD; + } + } +} + +#ifdef CONFIG_COMPAT +long spadfs_compat_ioctl(struct file *file, unsigned cmd, unsigned long arg) +{ + return spadfs_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); +} +#endif diff --git a/fs/spadfs/link.c b/fs/spadfs/link.c new file mode 100644 index 000000000..91f0d1902 --- /dev/null +++ b/fs/spadfs/link.c @@ -0,0 +1,127 @@ +#include "spadfs.h" + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,13) +static int spadfs_follow_link(struct dentry *dentry, struct nameidata *nd) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,2,0) +static void *spadfs_follow_link(struct dentry *dentry, struct nameidata *nd) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,5,0) +static const char *spadfs_follow_link(struct dentry *dentry, void **cookie) +#else +static const char *spadfs_follow_link(struct dentry *dentry, struct inode *inode, struct delayed_call *done) +#endif +{ + sync_lock_decl + SPADFNODE *f; + struct fnode_ea *lnk; + unsigned len; + char *str; + int r; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,5,0) + if (!dentry) + return ERR_PTR(-ECHILD); +#endif + + f = spadfnode(dentry->d_inode); + + down_read_sync_lock(f->fs); + mutex_lock(&f->file_lock); + + lnk = GET_EA((struct fnode_ea *)f->ea, f->ea_size, EA_SYMLINK_MAGIC, FNODE_EA_MAGIC_MASK); + + if (unlikely(!lnk)) { + spadfs_error(f->fs, TXFLAGS_EA_ERROR, + "can't find symlink extended attribute on fnode %Lx/%x", + (unsigned long long)f->fnode_block, + f->fnode_pos); + r = -EFSERROR; + goto ret_error; + } + + if (unlikely(lnk == GET_EA_ERROR)) { + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "error parsing extended attributes on symlink fnode %Lx/%x", + (unsigned long long)f->fnode_block, + f->fnode_pos); + r = -EFSERROR; + goto ret_error; + } + + len = SPAD2CPU32_LV(&lnk->magic) >> FNODE_EA_SIZE_SHIFT; + str = kmalloc(len + 1, GFP_NOFS); + if (unlikely(!str)) { + r = -ENOMEM; + goto ret_error; + } + + memcpy(str, lnk + 1, len); + str[len] = 0; +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,13) + nd_set_link(nd, str); + r = 0; +ret_error: + mutex_unlock(&f->file_lock); up_read_sync_lock(f->fs); + return r; +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,2,0) + nd_set_link(nd, str); + mutex_unlock(&f->file_lock); up_read_sync_lock(f->fs); + return str; +ret_error: + mutex_unlock(&f->file_lock); up_read_sync_lock(f->fs); + return ERR_PTR(r); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,5,0) + mutex_unlock(&f->file_lock); up_read_sync_lock(f->fs); + return *cookie = str; +ret_error: + mutex_unlock(&f->file_lock); up_read_sync_lock(f->fs); + return ERR_PTR(r); +#else + mutex_unlock(&f->file_lock); up_read_sync_lock(f->fs); + set_delayed_call(done, kfree_link, str); + return str; +ret_error: + mutex_unlock(&f->file_lock); up_read_sync_lock(f->fs); + return ERR_PTR(r); +#endif +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,13) +static void spadfs_put_link(struct dentry *dentry, struct nameidata *nd) +{ + char *str = nd_get_link(nd); + kfree(str); +} +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,2,0) +static void spadfs_put_link(struct dentry *dentry, struct nameidata *nd, void *str) +{ + kfree(str); +} +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21) +const +#endif +struct inode_operations spadfs_symlink_iops = { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) + .readlink = generic_readlink, +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,2,0) + .follow_link = spadfs_follow_link, + .put_link = spadfs_put_link, +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,5,0) + .follow_link = spadfs_follow_link, + .put_link = kfree_put_link, +#else + .get_link = spadfs_follow_link, +#endif + .setattr = spadfs_file_setattr, + .getattr = spadfs_getattr, +#ifdef SPADFS_XATTR +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,9,0) + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .removexattr = generic_removexattr, +#endif + .listxattr = spadfs_listxattr, +#endif +}; diff --git a/fs/spadfs/linux-compat.h b/fs/spadfs/linux-compat.h new file mode 100644 index 000000000..e183db8ee --- /dev/null +++ b/fs/spadfs/linux-compat.h @@ -0,0 +1,290 @@ +#ifndef _SPADFS_LINUX_COMPAT_H +#define _SPADFS_LINUX_COMPAT_H + +#define TEST_STABLE_BRANCH(v1,v2,v3) (LINUX_VERSION_CODE >= KERNEL_VERSION(v1,v2,v3) && LINUX_VERSION_CODE < KERNEL_VERSION(v1,(v2)+1,0)) + +#ifndef RHEL_MAJOR +#define TEST_RHEL_VERSION(v1,v2) 0 +#else +#define TEST_RHEL_VERSION(v1,v2) (RHEL_MAJOR == (v1) && RHEL_MINOR >= (v2)) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) +static inline void vprintk(const char *fmt, va_list args) +{ + char buffer[256]; + vsnprintf(buffer, sizeof buffer, fmt, args); + printk("%s", buffer); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,12) +static inline void *kmalloc_node(size_t size, int flags, int node) +{ + return kmalloc(size, flags); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,12) +static inline void cancel_rearming_delayed_workqueue( + struct workqueue_struct *wq, struct work_struct *work) +{ + while (!cancel_delayed_work(work)) + flush_workqueue(wq); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,13) +#define raw_smp_processor_id() smp_processor_id() +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,12) +#define rcu_barrier() synchronize_kernel() +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) +#define rcu_barrier() synchronize_rcu() +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) +static inline int rwsem_is_locked(struct rw_semaphore *s) +{ + if (down_write_trylock(s)) { + up_write(s); + return 0; + } + return 1; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,16) +#define mutex semaphore +#define mutex_init init_MUTEX +#define mutex_destroy(m) do { } while (0) +#define mutex_lock down +#define mutex_lock_nested(s, n) down(s) +#define mutex_unlock up +#define mutex_trylock(s) (!down_trylock(s)) +#define mutex_is_locked(s) (atomic_read(&(s)->count) < 1) +#define i_mutex i_sem +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17) +#define SLAB_MEM_SPREAD 0 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) && !TEST_RHEL_VERSION(5,3) +static inline void clear_nlink(struct inode *inode) +{ + inode->i_nlink = 0; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +typedef kmem_cache_t spadfs_cache_t; +#define spadfs_free_cache(c) do { if (c) { WARN_ON(kmem_cache_destroy(c)); (c) = NULL; } } while (0) +#else +typedef struct kmem_cache spadfs_cache_t; +#define spadfs_free_cache(c) do { if (c) { kmem_cache_destroy(c); (c) = NULL; } } while (0) +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,20) +#define NEW_WORKQUEUE +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +#define KMALLOC_MAX_SIZE 131072 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) +#define PTR_ALIGN(p, a) ((typeof(p))ALIGN((unsigned long)(p), (a))) +#endif + +#ifndef __cold +#define __cold +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25) && !TEST_RHEL_VERSION(5,3) +static inline int is_vmalloc_addr(const void *ptr) +{ +#ifdef CONFIG_MMU + if ((unsigned long)ptr >= VMALLOC_START && + (unsigned long)ptr < VMALLOC_END) + return 1; +#endif + return 0; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) && !TEST_RHEL_VERSION(5,4) +static inline size_t match_strlcpy(char *dest, const substring_t *src, size_t size) +{ + size_t ret = src->to - src->from; + + if (size) { + size_t len = ret >= size ? size - 1 : ret; + memcpy(dest, src->from, len); + dest[len] = '\0'; + } + + return ret; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) +#define get_current_uid() (current->uid) +#define get_current_gid() (current->gid) +#define get_current_fsuid() (current->fsuid) +#define get_current_fsgid() (current->fsgid) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) +#define get_current_uid() current_uid() +#define get_current_gid() current_gid() +#define get_current_fsuid() current_fsuid() +#define get_current_fsgid() current_fsgid() +#else +#define get_current_uid() from_kuid(&init_user_ns, current_uid()) +#define get_current_gid() from_kgid(&init_user_ns, current_gid()) +#define get_current_fsuid() from_kuid(&init_user_ns, current_fsuid()) +#define get_current_fsgid() from_kgid(&init_user_ns, current_fsgid()) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,28) +#define nr_cpumask_bits NR_CPUS +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,31) +static inline unsigned queue_max_sectors(struct request_queue *q) +{ + return q->max_sectors; +} +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,2,0) +#define HAVE_SET_NLINK +#endif +#if TEST_RHEL_VERSION(6,9) +#if RHEL_RELEASE >= 753 +#define HAVE_SET_NLINK +#endif +#endif +#ifndef HAVE_SET_NLINK +static inline void set_nlink(struct inode *inode, unsigned nlink) +{ + inode->i_nlink = nlink; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) +#define i_uid_read(i) ((i)->i_uid) +#define i_gid_read(i) ((i)->i_gid) +#define i_uid_write(i, u) ((i)->i_uid = (u)) +#define i_gid_write(i, u) ((i)->i_gid = (u)) +#define uid_eq(a, b) ((a) == (b)) +#define gid_eq(a, b) ((a) == (b)) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) +#define file_inode(f) (file_dentry(f)->d_inode) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,15,0) && !TEST_STABLE_BRANCH(3,12,41) && !TEST_RHEL_VERSION(6,8) && !TEST_RHEL_VERSION(7,1) +static inline void kvfree(void *ptr) +{ + if (is_vmalloc_addr(ptr)) + vfree(ptr); + else + kfree(ptr); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0) +#define ktime_get_real_seconds get_seconds +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0) +#define file_dentry(f) ((f)->f_dentry) +#elif !(LINUX_VERSION_CODE >= KERNEL_VERSION(4,5,2) || (LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,8) && LINUX_VERSION_CODE < KERNEL_VERSION(4,5,0))) +#define file_dentry(f) ((f)->f_path.dentry) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,4,0) +#define __GFP_DIRECT_RECLAIM __GFP_WAIT +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,0) +#define huge_valid_dev(dev) 1 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,4,21) && !TEST_RHEL_VERSION(7,4) +/* warning: do not define inode_lock, it collides with another symbol on kernels <= 2.6.38 */ +static inline void inode_unlock(struct inode *inode) +{ + mutex_unlock(&inode->i_mutex); +} +static inline int inode_trylock(struct inode *inode) +{ + return mutex_trylock(&inode->i_mutex); +} +static inline int inode_is_locked(struct inode *inode) +{ + return mutex_is_locked(&inode->i_mutex); +} +static inline void inode_lock_nested(struct inode *inode, unsigned subclass) +{ + mutex_lock_nested(&inode->i_mutex, subclass); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,5,0) && !defined(SLAB_ACCOUNT) +#define SLAB_ACCOUNT 0 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +#define bio_set_dev(bio, dev) ((bio)->bi_bdev = (dev)) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +#define SB_RDONLY MS_RDONLY +#define SB_NOATIME MS_NOATIME +static inline int sb_rdonly(const struct super_block *sb) { return (sb->s_flags & SB_RDONLY) != 0; } +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) +#define time_t u32 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,3,0) +#define mnt_idmap user_namespace +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,6,0) +#define inode_get_ctime(inode) ((inode)->i_ctime) +static inline void inode_set_ctime(struct inode *inode, __u64 sec, long nsec) +{ + inode->i_ctime.tv_sec = sec; + inode->i_ctime.tv_nsec = nsec; +} +#endif + +#ifndef READ_ONCE +#if defined(ACCESS_ONCE) +#define READ_ONCE ACCESS_ONCE +#else +#define READ_ONCE(x) (*(volatile typeof(x) *)&(x)) +#endif +#endif + +#ifndef WRITE_ONCE +#if defined(ACCESS_ONCE_RW) +/* grsecurity hack */ +#define WRITE_ONCE(x, y) (ACCESS_ONCE_RW(x) = (y)) +#elif defined(ACCESS_ONCE) +#define WRITE_ONCE(x, y) (ACCESS_ONCE(x) = (y)) +#else +#define WRITE_ONCE(x, y) (*(volatile typeof(x) *)&(x)) = (y) +#endif +#endif + +#ifndef BIO_MAX_VECS +#define BIO_MAX_VECS BIO_MAX_PAGES +#endif + +#endif diff --git a/fs/spadfs/name.c b/fs/spadfs/name.c new file mode 100644 index 000000000..78483db39 --- /dev/null +++ b/fs/spadfs/name.c @@ -0,0 +1,109 @@ +#include "spadfs.h" + +/* 3.8.11 is the chromebook kernel */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0) && LINUX_VERSION_CODE != KERNEL_VERSION(3,8,11) && !TEST_RHEL_VERSION(7,0) +#define PASS_INODE +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) +static int spadfs_hash_dentry(struct dentry *dentry, struct qstr *qstr) +#else +static int spadfs_hash_dentry(const struct dentry *dentry, +#ifdef PASS_INODE + const struct inode *inode, +#endif + struct qstr *qstr) +#endif +{ + unsigned i; + unsigned long hash; + /*printk("hash '%.*s'\n", qstr->len, qstr->name);*/ + if (unlikely((unsigned)(qstr->len - 1) > (MAX_NAME_LEN - 1))) + return -ENAMETOOLONG; + if (unlikely(qstr->name[0] == '^')) + return -EINVAL; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,8,0) + hash = init_name_hash(); +#else + hash = init_name_hash(dentry); +#endif + for (i = 0; i < qstr->len; i++) { + if (unlikely(qstr->name[i] == ':')) + return -EINVAL; + hash = partial_name_hash(qstr->name[i] & 0xdf, hash); + } + qstr->hash = hash; + return 0; +} + +static int spadfs_compare_names_internal(int unx, const char *n1, unsigned l1, + const char *n2, unsigned l2) +{ + if (l1 != l2) + return 1; + if (likely(unx)) { + return memcmp(n1, n2, l1); + } else { + while (l1--) { + char c1 = *n1++; + char c2 = *n2++; + if (c1 >= 'a' && c1 <= 'z') c1 -= 0x20; + if (c2 >= 'a' && c2 <= 'z') c2 -= 0x20; + if (c1 != c2) return 1; + } + return 0; + } +} + +int spadfs_compare_names(SPADFS *fs, const char *n1, unsigned l1, + const char *n2, unsigned l2) +{ + return spadfs_compare_names_internal(!!(fs->flags_compat_none & FLAG_COMPAT_NONE_UNIX_NAMES), n1, l1, n2, l2); +} + +void spadfs_set_name(SPADFS *fs, char *dest, const char *src, unsigned len) +{ + strncpy(dest, src, (len + 7) & ~7); + if (unlikely(!(fs->flags_compat_none & FLAG_COMPAT_NONE_UNIX_NAMES))) { + while (len--) { + if (*dest >= 'a' && *dest <= 'z') *dest -= 0x20; + dest++; + } + } +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) +static int spadfs_compare_dentry(struct dentry *dentry, + struct qstr *a, struct qstr *b) +{ + return spadfs_compare_names_internal(0, + (const char *)a->name, a->len, + (const char *)b->name, b->len); +} +#else +static int spadfs_compare_dentry( +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,8,0) + const struct dentry *p_dentry, +#endif +#ifdef PASS_INODE + const struct inode *p_inode, +#endif + const struct dentry *dentry, +#ifdef PASS_INODE + const struct inode *inode, +#endif + unsigned len, const char *a, + const struct qstr *b) +{ + /*printk("compare '%.*s', '%.*s'\n", len, a, b->len, b->name);*/ + return spadfs_compare_names_internal(0, a, len, b->name, b->len); +} +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30) +const +#endif +struct dentry_operations spadfs_dops = { + .d_hash = spadfs_hash_dentry, + .d_compare = spadfs_compare_dentry, +}; diff --git a/fs/spadfs/namei.c b/fs/spadfs/namei.c new file mode 100644 index 000000000..09864ce26 --- /dev/null +++ b/fs/spadfs/namei.c @@ -0,0 +1,1204 @@ +#include "spadfs.h" +#include "dir.h" + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +#define mode_type int +#else +#define mode_type umode_t +#endif + +static struct dentry *spadfs_lookup(struct inode *dir, struct dentry *dentry, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) + struct nameidata *nd +#else + unsigned flags +#endif + ) +{ + sync_lock_decl + int r; + spadfs_ino_t ino; + struct inode *result; + SPADFS *fs = spadfnode(dir)->fs; + + if (unlikely(dentry->d_name.len > MAX_NAME_LEN)) + return ERR_PTR(-ENAMETOOLONG); + if (unlikely(is_deleted_file(spadfnode(dir)))) + return ERR_PTR(-ENOENT); + + down_read_sync_lock(fs); + + r = spadfs_lookup_ino(spadfnode(dir), &dentry->d_name, &ino, 0); + if (unlikely(r)) { + up_read_sync_lock(fs); + if (r < 0) + return ERR_PTR(r); + goto not_found; + } + + result = spadfs_iget(dir->i_sb, ino, spadfnode(dir)->fnode_block, + spadfnode(dir)->fnode_pos); + up_read_sync_lock(fs); + if (unlikely(IS_ERR(result))) + return ERR_PTR(PTR_ERR(result)); + + spadfs_set_dentry_operations(fs, dentry); + d_add(dentry, result); + return NULL; + +not_found: + spadfs_set_dentry_operations(fs, dentry); + d_add(dentry, NULL); + return NULL; +} + +static int spadfs_get_dirent_args(struct fnode *fnode, unsigned size, + unsigned namelen, u64 *ino, unsigned *dt) +{ + struct fnode_ea *ea, *eas; + unsigned ea_size; + + *dt = DT_REG; + + eas = (struct fnode_ea *)((char *)fnode + FNODE_EA_POS(namelen)); + ea_size = size - FNODE_EA_POS(namelen); + + ea = GET_EA(eas, ea_size, EA_UNX_MAGIC, EA_UNX_MAGIC_MASK); + if (unlikely(!ea)) { + ea = GET_EA(eas, ea_size, EA_SYMLINK_MAGIC, + EA_SYMLINK_MAGIC_MASK); + if (unlikely(ea == GET_EA_ERROR)) + return -EFSERROR; + + if (likely(ea != NULL)) { + *dt = DT_LNK; + spadfs_validate_stable_ino(ino, + ((u64)SPAD2CPU32_LV(&fnode->run10)) | + ((u64)SPAD2CPU16_LV(&fnode->run11) << 32) | + ((u64)SPAD2CPU16_LV(&fnode->run1n) << 48)); + } + } else if (unlikely(ea == GET_EA_ERROR)) { + return -EFSERROR; + } else { + /* + * Dirent type is documented in include/linux/fs.h, + * so I hope it won't change + */ + *dt = (SPAD2CPU16_LV(&((struct ea_unx *)ea)->mode) >> 12) & 15; + if (likely(SPAD2CPU32_LV(&ea->magic) == EA_UNX_MAGIC)) + spadfs_validate_stable_ino(ino, + SPAD2CPU64_LV(&((struct ea_unx *)ea)->ino)); + } + + if (unlikely(fnode->flags & FNODE_FLAGS_DIR)) + *dt = DT_DIR; + + return 0; +} + +static noinline int spadfs_get_hardlink_args(SPADFS *fs, + sector_t fixed_fnode_sec, + u64 *ino, unsigned *dt) +{ + int r; + struct fnode_block *fixed_fnode_block; + struct buffer_head *fbh; + unsigned ffpos; + struct fnode *fixed_fnode; + unsigned size, namelen; + + fixed_fnode_block = spadfs_read_fnode_block(fs, fixed_fnode_sec, &fbh, + SRFB_FIXED_FNODE, "spadfs_get_hardlink_args"); + if (unlikely(IS_ERR(fixed_fnode_block))) { + r = PTR_ERR(fixed_fnode_block); + goto brelse_ret; + } + + r = spadfs_get_fixed_fnode_pos(fs, fixed_fnode_block, fixed_fnode_sec, + &ffpos); + if (unlikely(r)) + goto brelse_ret_r; + + fixed_fnode = (struct fnode *)((char *)fixed_fnode_block + ffpos); + + size = SPAD2CPU16_LV(&fixed_fnode->next) & FNODE_NEXT_SIZE; + namelen = fixed_fnode->namelen; + + if (unlikely(size < FNODE_EA_POS(namelen)) || + unlikely(ffpos + size > FNODE_BLOCK_SIZE)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "invalid size in fixed fnode %Lx", + (unsigned long long)fixed_fnode_sec); + r = -EFSERROR; + goto brelse_ret_r; + } + + r = spadfs_get_dirent_args(fixed_fnode, size, namelen, ino, dt); + +brelse_ret_r: + spadfs_brelse(fs, fbh); +brelse_ret: + return r; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0) +static int spadfs_readdir(struct file *file, void *dirent, filldir_t filldir) +#define f_pos file->f_pos +#define emit(name, len, ino, type) filldir(dirent, name, len, f_pos, ino, type) +#else +static int spadfs_readdir(struct file *file, struct dir_context *ctx) +#define f_pos ctx->pos +#define emit(name, len, ino, type) (!dir_emit(ctx, name, len, ino, type)) +#endif +{ + sync_lock_decl + int r; + SPADFNODE *f; + struct buffer_head *bh; + struct fnode_block *fnode_block; + sector_t secno; + hash_t hash, next_hash; + unsigned order, pos; + unsigned current_order; + sector_t c[2]; + c[1] = 0; + f = spadfnode(file_inode(file)); + + down_read_sync_lock(f->fs); + + if (unlikely(is_dir_off_t_special(f_pos))) { + unsigned n = dir_off_t_special_n(f_pos); + switch (n) { + case 0: + if (unlikely(emit(".", 1, + spadfs_get_user_inode_number(file_inode(file)), + DT_DIR))) + goto ret_0; + f_pos = make_special_dir_off_t(1); + goto label_1; + case 1: + label_1: + if (unlikely(emit("..", 2, + spadfs_get_user_inode_number( + file_dentry(file)->d_parent-> + d_inode), + DT_DIR))) + goto ret_0; + f_pos = make_dir_off_t(0, 0, + SIZEOF_FNODE_BLOCK); + break; + case 2: + goto ret_0; + case 3: + r = -EFSERROR; + goto ret_r; + default: + r = -EINVAL; + goto ret_r; + } + } + + if (unlikely(is_deleted_file(f))) + goto eof; + + hash = dir_off_t_hash(f_pos); + order = dir_off_t_order(f_pos); + pos = dir_off_t_pos(f_pos); + +new_hash_lookup: + fnode_block = spadfs_find_hash_block(f, hash, &bh, &secno, &next_hash); + current_order = 0; + if (unlikely(!fnode_block)) + goto use_next_hash; + +next_fnode_block: + if (unlikely(IS_ERR(fnode_block))) { + r = PTR_ERR(fnode_block); + goto ret_r; + } + if (likely(current_order >= order)) { + unsigned size, namelen, fpos; + struct fnode *fnode = fnode_block->fnodes; +next_fnode: + VALIDATE_FNODE(f->fs, fnode_block, fnode, size, namelen, + ok, skip, bad_fnode); + +ok: + fpos = (char *)fnode - (char *)fnode_block; + if (likely(fpos >= pos)) { + u64 ino; + unsigned dt; + f_pos = make_dir_off_t(hash, current_order, fpos); + if (likely(!(fnode->flags & FNODE_FLAGS_HARDLINK))) { + ino = spadfs_ino_t_2_ino64_t( + make_spadfs_ino_t(secno, fpos)); + if (unlikely(spadfs_get_dirent_args(fnode, size, + namelen, &ino, &dt))) { + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "error parsing extended attributes on fnode %Lx/%x during readdir of %Lx/%x", + (unsigned long long)secno, + fpos, + (unsigned long long)f->fnode_block, + f->fnode_pos); + goto brelse_ret_efserror; + } + } else { + sector_t fixed_fnode_sec = MAKE_D_OFF( + fnode->anode0, fnode->anode1); + ino = spadfs_ino_t_2_ino64_t( + make_fixed_spadfs_ino_t(fixed_fnode_sec)); + dt = DT_UNKNOWN; + r = spadfs_get_hardlink_args(f->fs, + fixed_fnode_sec, &ino, &dt); + if (unlikely(r)) + goto brelse_ret_r; + } + ino = spadfs_expose_inode_number(f->fs, ino); + if (unlikely(emit(FNODE_NAME(fnode), namelen, + ino, dt))) { + spadfs_brelse(f->fs, bh); + goto ret_0; + } + } + +skip: + fnode = (struct fnode *)((char *)fnode + size); + if (likely(((unsigned long)fnode & (FNODE_BLOCK_SIZE - 1)) != + 0)) goto next_fnode; + pos = SIZEOF_FNODE_BLOCK; + } + + if (!(fnode_block->flags & FNODE_BLOCK_LAST)) { + spadfs_brelse(f->fs, bh); + secno++; +read_next_fnode_block: + if (unlikely(spadfs_stop_cycles(f->fs, secno, &c, + "spadfs_readdir"))) { + r = -EFSERROR; + goto ret_r; + } + current_order++; + fnode_block = spadfs_read_fnode_block(f->fs, secno, &bh, + SRFB_FNODE, "spadfs_readdir"); + goto next_fnode_block; + } + if (unlikely(CC_VALID(f->fs, &fnode_block->cc, &fnode_block->txc))) { + secno = MAKE_D_OFF(fnode_block->next0, fnode_block->next1); + spadfs_brelse(f->fs, bh); + goto read_next_fnode_block; + } + spadfs_brelse(f->fs, bh); + +use_next_hash: + if (unlikely(next_hash != 0)) { + order = 0; + hash = next_hash; + goto new_hash_lookup; + } + +eof: + f_pos = make_special_dir_off_t(2); +ret_0: + up_read_sync_lock(f->fs); + return 0; + +bad_fnode: + spadfs_error(f->fs, TXFLAGS_FS_ERROR, + "bad fnode on block %Lx when reading directory", + (unsigned long long)secno); +brelse_ret_efserror: + r = -EFSERROR; +brelse_ret_r: + spadfs_brelse(f->fs, bh); +ret_r: + /* + * On error, we must set invalid f_pos. Otherwise, Linux + * would loop on the erroneous entry forever. + */ + f_pos = make_special_dir_off_t(3); + up_read_sync_lock(f->fs); + return r; +#undef f_pos +#undef emit +} + +static void spadfs_update_directory_times(struct inode *dir) +{ + time_t t = ktime_get_real_seconds(); + if (likely(t == dir->i_mtime.tv_sec) && + likely(t == inode_get_ctime(dir).tv_sec)) + return; + dir->i_mtime.tv_sec = t; + dir->i_mtime.tv_nsec = 0; + inode_set_ctime(dir, t, 0); + spadfs_write_directory(spadfnode(dir)); +} + +static int spadfs_create( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) + struct mnt_idmap *ns, +#endif + struct inode *dir, struct dentry *dentry, + mode_type mode, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) + struct nameidata *nd +#else + bool excl +#endif + ) +{ + sync_lock_decl + SPADFS *fs = spadfnode(dir)->fs; + struct buffer_head *bh; + sector_t fnode_address; + unsigned fnode_off; + struct fnode_block *fnode_block; + struct fnode *fnode; + int wlock = 0; + int synced = 0; + time_t ctime; + struct inode *inode; + int r; + struct ea_unx *ea; + u64 stable_ino; + +#ifdef SPADFS_QUOTA + dquot_initialize(dir); +#endif + + r = spadfs_new_stable_ino(fs, &stable_ino); + if (unlikely(r)) + goto ret_r; + +lock_and_again: + ND_LOCK(fs, wlock); + if (unlikely(is_deleted_file(spadfnode(dir)))) { + r = -ENOENT; + goto unlock_ret_r; + } + + fnode = spadfs_add_fnode_to_directory(spadfnode(dir), + (const char *)dentry->d_name.name, dentry->d_name.len, + FNODE_EA_DO_ALIGN(sizeof(struct ea_unx)), + &bh, &fnode_address, &fnode_off, &fnode_block, wlock); + if (unlikely(IS_ERR(fnode))) { + if (likely(fnode == ERR_PTR(-ENOSPC)) && !synced) + goto unlock_do_sync; + r = PTR_ERR(fnode); + goto unlock_ret_r; + } + if (unlikely(fnode == NEED_SYNC)) + goto unlock_do_sync; + if (unlikely(fnode == NEED_WLOCK)) { + BUG_ON(wlock); + ND_UNLOCK(fs, wlock); + wlock = 1; + goto lock_and_again; + } + + ctime = ktime_get_real_seconds(); + fnode->size[0] = CPU2SPAD64_CONST(0); + fnode->size[1] = CPU2SPAD64_CONST(0); + fnode->ctime = fnode->mtime = CPU2SPAD32((u32)ctime); + fnode->run10 = MAKE_PART_0(0); + fnode->run11 = MAKE_PART_1(0); + fnode->run1n = CPU2SPAD16_CONST(0); + fnode->run20 = MAKE_PART_0(0); + fnode->run21 = MAKE_PART_1(0); + fnode->run2n = CPU2SPAD16_CONST(0); + fnode->anode0 = MAKE_PART_0(0); + fnode->anode1 = MAKE_PART_1(0); + fnode->flags = 0; + fnode->namelen = dentry->d_name.len; + spadfs_set_name(fs, FNODE_NAME(fnode), + (const char *)dentry->d_name.name, dentry->d_name.len); + ea = (struct ea_unx *)((char *)fnode + + FNODE_EA_POS(dentry->d_name.len)); + spadfs_init_unx_attribute(dir, ea, mode, stable_ino); + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + + inode = spadfs_new_inode(dir, mode, ctime, ea, + FNODE_EA_DO_ALIGN(sizeof(struct ea_unx)), + fnode_address, fnode_off, 0, stable_ino); + spadfs_brelse(fs, bh); + if (unlikely(IS_ERR(inode))) { + r = PTR_ERR(inode); + goto remove_entry_unlock_ret_r; + } + + spadfs_update_directory_times(dir); + d_instantiate(dentry, inode); + ND_UNLOCK(fs, wlock); + return 0; + +unlock_do_sync: + ND_UNLOCK(fs, wlock); + synced = 1; + r = spadfs_commit(fs); + if (unlikely(r)) + return r; + goto lock_and_again; + +remove_entry_unlock_ret_r: + spadfs_remove_fnode_from_directory(spadfnode(dir), NULL, + &dentry->d_name); +unlock_ret_r: + ND_UNLOCK(fs, wlock); +ret_r: + return r; +} + +static int spadfs_mkdir( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) + struct mnt_idmap *ns, +#endif + struct inode *dir, struct dentry *dentry, + mode_type mode) +{ + sync_lock_decl + SPADFS *fs = spadfnode(dir)->fs; + struct buffer_head *bh; + sector_t fnode_address; + unsigned fnode_off; + struct fnode_block *fnode_block; + struct fnode *fnode; + int wlock = 0; + int synced = 0; + time_t ctime; + struct inode *inode; + int r; + struct ea_unx *ea; + sector_t new_dir; + u16 hint_small, hint_big; + u64 stable_ino; + +#ifdef SPADFS_QUOTA + dquot_initialize(dir); +#endif + + r = spadfs_new_stable_ino(fs, &stable_ino); + if (unlikely(r)) + goto ret_r; + + mode |= S_IFDIR; + +lock_and_again: + ND_LOCK(fs, wlock); + /* + * if hint reading were after spadfs_add_fnode_to_directory, + * it could deadlock + */ + spadfs_get_dir_hint(spadfnode(dir), &hint_small, &hint_big); + + if (unlikely(is_deleted_file(spadfnode(dir)))) { + ND_UNLOCK(fs, wlock); + r = -ENOENT; + goto ret_r; + } + r = spadfs_alloc_leaf_page(spadfnode(dir), spadfnode(dir)->fnode_block, 1U << fs->sectors_per_disk_block_bits, 0, &new_dir, 1); + if (unlikely(r)) { + if (likely(r == -ENOSPC) && !synced) + goto unlock_do_sync; + goto unlock_ret_r; + } + fnode = spadfs_add_fnode_to_directory(spadfnode(dir), + (const char *)dentry->d_name.name, dentry->d_name.len, + FNODE_EA_DO_ALIGN(sizeof(struct ea_unx)), &bh, + &fnode_address, &fnode_off, &fnode_block, wlock); + if (unlikely(IS_ERR(fnode))) { + r = spadfs_free_blocks_metadata(fs, new_dir, 1U << fs->sectors_per_disk_block_bits); + if (unlikely(r)) + goto unlock_ret_r; + if (likely(fnode == ERR_PTR(-ENOSPC)) && !synced) + goto unlock_do_sync; + r = PTR_ERR(fnode); + goto unlock_ret_r; + } + if (unlikely(fnode == NEED_SYNC)) { + r = spadfs_free_blocks_metadata(fs, new_dir, 1U << fs->sectors_per_disk_block_bits); + if (unlikely(r)) + goto unlock_ret_r; + goto unlock_do_sync; + } + if (unlikely(fnode == NEED_WLOCK)) { + BUG_ON(wlock); + r = spadfs_free_blocks_metadata(fs, new_dir, 1U << fs->sectors_per_disk_block_bits); + if (unlikely(r)) + goto unlock_ret_r; + ND_UNLOCK(fs, wlock); + wlock = 1; + goto lock_and_again; + } + + ctime = ktime_get_real_seconds(); + fnode->size[0] = fnode->size[1] = CPU2SPAD64(directory_size(fs) ? 512U << fs->sectors_per_disk_block_bits : 0); + fnode->ctime = fnode->mtime = CPU2SPAD32((u32)ctime); + fnode->run10 = MAKE_PART_0(new_dir); + fnode->run11 = MAKE_PART_1(new_dir); + fnode->run1n = CPU2SPAD16(hint_small); + fnode->run20 = MAKE_PART_0(new_dir); + fnode->run21 = MAKE_PART_1(new_dir); + fnode->run2n = CPU2SPAD16(hint_big); + fnode->anode0 = MAKE_PART_0(0); + fnode->anode1 = MAKE_PART_1(0); + fnode->flags = FNODE_FLAGS_DIR; + fnode->namelen = dentry->d_name.len; + spadfs_set_name(fs, FNODE_NAME(fnode), + (const char *)dentry->d_name.name, dentry->d_name.len); + ea = (struct ea_unx *)((char *)fnode + + FNODE_EA_POS(dentry->d_name.len)); + spadfs_init_unx_attribute(dir, ea, mode, stable_ino); + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + + inode = spadfs_new_inode(dir, mode, ctime, ea, + FNODE_EA_DO_ALIGN(sizeof(struct ea_unx)), + fnode_address, fnode_off, new_dir, stable_ino); + spadfs_brelse(fs, bh); + if (unlikely(IS_ERR(inode))) { + r = PTR_ERR(inode); + goto remove_entry_unlock_ret_r; + } + + spadfs_update_directory_times(dir); + d_instantiate(dentry, inode); + ND_UNLOCK(fs, wlock); + return 0; + +unlock_do_sync: + ND_UNLOCK(fs, wlock); + synced = 1; + r = spadfs_commit(fs); + if (unlikely(r)) + return r; + goto lock_and_again; + +remove_entry_unlock_ret_r: + spadfs_remove_fnode_from_directory(spadfnode(dir), NULL, &dentry->d_name); + spadfs_free_blocks_metadata(fs, new_dir, 1U << fs->sectors_per_disk_block_bits); +unlock_ret_r: + ND_UNLOCK(fs, wlock); +ret_r: + return r; +} + +static int spadfs_mknod( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) + struct mnt_idmap *ns, +#endif + struct inode *dir, struct dentry *dentry, + mode_type mode, dev_t rdev) +{ + sync_lock_decl + SPADFS *fs = spadfnode(dir)->fs; + struct buffer_head *bh; + sector_t fnode_address; + unsigned fnode_off; + struct fnode_block *fnode_block; + struct fnode *fnode; + int wlock = 0; + int synced = 0; + time_t ctime; + struct inode *inode; + int r; + struct ea_unx *ea; + int rdev_needed; + u64 stable_ino; + +#ifdef SPADFS_QUOTA + dquot_initialize(dir); +#endif + + r = spadfs_new_stable_ino(fs, &stable_ino); + if (unlikely(r)) + goto ret_r; + + if (likely(S_ISFIFO(mode)) || likely(S_ISSOCK(mode))) { + rdev_needed = 0; + } else if (unlikely(S_ISBLK(mode)) || unlikely(S_ISCHR(mode))) { + rdev_needed = 1; + if (unlikely(!huge_valid_dev(rdev))) + return -EINVAL; + } else { + return -EINVAL; + } + +lock_and_again: + ND_LOCK(fs, wlock); + if (unlikely(is_deleted_file(spadfnode(dir)))) { + r = -ENOENT; + goto unlock_ret_r; + } + + fnode = spadfs_add_fnode_to_directory(spadfnode(dir), + (const char *)dentry->d_name.name, dentry->d_name.len, + FNODE_EA_DO_ALIGN(sizeof(struct ea_unx)) + + (rdev_needed ? + FNODE_EA_DO_ALIGN(sizeof(struct ea_rdev)) : 0), + &bh, &fnode_address, &fnode_off, &fnode_block, wlock); + if (unlikely(IS_ERR(fnode))) { + if (likely(fnode == ERR_PTR(-ENOSPC)) && !synced) + goto unlock_do_sync; + r = PTR_ERR(fnode); + goto unlock_ret_r; + } + if (unlikely(fnode == NEED_SYNC)) + goto unlock_do_sync; + if (unlikely(fnode == NEED_WLOCK)) { + BUG_ON(wlock); + ND_UNLOCK(fs, wlock); + wlock = 1; + goto lock_and_again; + } + + ctime = ktime_get_real_seconds(); + fnode->size[0] = CPU2SPAD64_CONST(0); + fnode->size[1] = CPU2SPAD64_CONST(0); + fnode->ctime = fnode->mtime = CPU2SPAD32((u32)ctime); + fnode->run10 = MAKE_PART_0(0); + fnode->run11 = MAKE_PART_1(0); + fnode->run1n = CPU2SPAD16_CONST(0); + fnode->run20 = MAKE_PART_0(0); + fnode->run21 = MAKE_PART_1(0); + fnode->run2n = CPU2SPAD16_CONST(0); + fnode->anode0 = MAKE_PART_0(0); + fnode->anode1 = MAKE_PART_1(0); + fnode->flags = 0; + fnode->namelen = dentry->d_name.len; + spadfs_set_name(fs, FNODE_NAME(fnode), + (const char *)dentry->d_name.name, dentry->d_name.len); + ea = (struct ea_unx *)((char *)fnode + + FNODE_EA_POS(dentry->d_name.len)); + spadfs_init_unx_attribute(dir, ea, mode, stable_ino); + if (rdev_needed) { + struct ea_rdev *rd = (struct ea_rdev *)((char *)ea + + FNODE_EA_DO_ALIGN(sizeof(struct ea_unx))); + CPU2SPAD32_LV(&rd->magic, EA_RDEV_MAGIC); + CPU2SPAD32_LV(&rd->pad, 0); + CPU2SPAD64_LV(&rd->dev, huge_encode_dev(rdev)); + } + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + + inode = spadfs_new_inode(dir, S_IFREG, ctime, ea, + FNODE_EA_DO_ALIGN(sizeof(struct ea_unx)) + + (rdev_needed ? + FNODE_EA_DO_ALIGN(sizeof(struct ea_rdev)) : + 0), + fnode_address, fnode_off, 0, stable_ino); + spadfs_brelse(fs, bh); + if (unlikely(IS_ERR(inode))) { + r = PTR_ERR(inode); + goto remove_entry_unlock_ret_r; + } + + spadfs_update_directory_times(dir); + d_instantiate(dentry, inode); + ND_UNLOCK(fs, wlock); + return 0; + +unlock_do_sync: + ND_UNLOCK(fs, wlock); + synced = 1; + r = spadfs_commit(fs); + if (unlikely(r)) + return r; + goto lock_and_again; + +remove_entry_unlock_ret_r: + spadfs_remove_fnode_from_directory(spadfnode(dir), NULL, + &dentry->d_name); +unlock_ret_r: + ND_UNLOCK(fs, wlock); +ret_r: + return r; +} + +static int spadfs_symlink( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) + struct mnt_idmap *ns, +#endif + struct inode *dir, struct dentry *dentry, + const char *symlink) +{ + sync_lock_decl + SPADFS *fs = spadfnode(dir)->fs; + struct buffer_head *bh; + sector_t fnode_address; + unsigned fnode_off; + struct fnode_block *fnode_block; + struct fnode *fnode; + int wlock = 0; + int synced = 0; + time_t ctime; + struct inode *inode; + int r; + struct fnode_ea *ea; + unsigned strlen_symlink; + unsigned ea_size; + u64 stable_ino; + umode_t mode; + uid_t uid; + gid_t gid; + +#ifdef SPADFS_QUOTA + dquot_initialize(dir); +#endif + + r = spadfs_new_stable_ino(fs, &stable_ino); + if (unlikely(r)) + goto ret_r; + + mode = S_IFLNK | S_IRWXUGO; + spadfs_get_initial_attributes(dir, &mode, &uid, &gid); + + strlen_symlink = strlen(symlink); + ea_size = FNODE_EA_DO_ALIGN(sizeof(struct fnode_ea) + strlen_symlink); + if (unlikely(ea_size > FNODE_MAX_EA_SIZE)) + return -ENAMETOOLONG; + +lock_and_again: + ND_LOCK(fs, wlock); + if (unlikely(is_deleted_file(spadfnode(dir)))) { + r = -ENOENT; + goto unlock_ret_r; + } + fnode = spadfs_add_fnode_to_directory(spadfnode(dir), + (const char *)dentry->d_name.name, dentry->d_name.len, + ea_size, + &bh, &fnode_address, &fnode_off, &fnode_block, wlock); + if (unlikely(IS_ERR(fnode))) { + if (likely(fnode == ERR_PTR(-ENOSPC)) && !synced) + goto unlock_do_sync; + r = PTR_ERR(fnode); + goto unlock_ret_r; + } + if (unlikely(fnode == NEED_SYNC)) + goto unlock_do_sync; + if (unlikely(fnode == NEED_WLOCK)) { + BUG_ON(wlock); + ND_UNLOCK(fs, wlock); + wlock = 1; + goto lock_and_again; + } + ctime = ktime_get_real_seconds(); + fnode->size[0] = CPU2SPAD64_CONST(0); + fnode->size[1] = CPU2SPAD64_CONST(0); + fnode->ctime = fnode->mtime = CPU2SPAD32((u32)ctime); + fnode->run10 = CPU2SPAD32(stable_ino); + fnode->run11 = CPU2SPAD16(stable_ino >> 32); + fnode->run1n = CPU2SPAD16(stable_ino >> 48); + fnode->run20 = CPU2SPAD32(uid); + fnode->run21 = CPU2SPAD16(gid); + fnode->run2n = CPU2SPAD16(gid >> 16); + fnode->anode0 = MAKE_PART_0(0); + fnode->anode1 = MAKE_PART_1(0); + fnode->flags = 0; + fnode->namelen = dentry->d_name.len; + spadfs_set_name(fs, FNODE_NAME(fnode), + (const char *)dentry->d_name.name, dentry->d_name.len); + ea = (struct fnode_ea *)((char *)fnode + + FNODE_EA_POS(dentry->d_name.len)); + CPU2SPAD32_LV(&ea->magic, + EA_SYMLINK_MAGIC | (strlen_symlink << FNODE_EA_SIZE_SHIFT)); + strncpy((char *)(ea + 1), symlink, ea_size - sizeof(struct fnode_ea)); + do_fnode_block_checksum(fs, fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + + inode = spadfs_new_inode(dir, S_IFREG, ctime, ea, ea_size, + fnode_address, fnode_off, 0, stable_ino); + spadfs_brelse(fs, bh); + if (unlikely(IS_ERR(inode))) { + r = PTR_ERR(inode); + goto remove_entry_unlock_ret_r; + } + i_uid_write(inode, uid); + i_gid_write(inode, gid); + spadfs_update_directory_times(dir); + d_instantiate(dentry, inode); + ND_UNLOCK(fs, wlock); + return 0; + +unlock_do_sync: + ND_UNLOCK(fs, wlock); + synced = 1; + r = spadfs_commit(fs); + if (unlikely(r)) + return r; + goto lock_and_again; + +remove_entry_unlock_ret_r: + spadfs_remove_fnode_from_directory(spadfnode(dir), NULL, + &dentry->d_name); +unlock_ret_r: + ND_UNLOCK(fs, wlock); +ret_r: + return r; +} + +static int spadfs_link(struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry) +{ + /*sync_lock_decl*/ + struct inode *file = old_dentry->d_inode; + struct inode *old_dir = old_dentry->d_parent->d_inode; + SPADFS *fs = spadfnode(file)->fs; + struct buffer_head *bh; + sector_t link_address; + unsigned link_off; + struct fnode_block *link_fnode_block; + struct fnode *fnode; + int synced = 0; + int r; + +#ifdef SPADFS_QUOTA + dquot_initialize(new_dir); +#endif + +lock_and_again: + down_write_sync_lock(fs); + if (!is_fnode_fixed(spadfnode(file))) { + /* + * This is the first link --- + * we need to move file to its own new block + */ + sector_t new_fxblk; + u16 hint_small, hint_big; + spadfs_get_dir_hint(spadfnode(old_dir), &hint_small, &hint_big); + if (unlikely(spadfnode(file)->spadfs_nlink > 1)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "file has more links but is in directory"); + r = -EFSERROR; + goto unlock_ret_r; + } + + r = spadfs_alloc_fixed_fnode_block(spadfnode(old_dir), + spadfnode(old_dir)->fnode_block, + FNODE_SIZE(0, spadfnode(file)->ea_size), + hint_small, hint_big, &new_fxblk); + if (unlikely(r)) { + if (likely(r == -ENOSPC && !synced)) + goto unlock_do_sync; + goto unlock_ret_r; + } + + fnode = spadfs_add_fnode_to_directory(spadfnode(old_dir), + (const char *)old_dentry->d_name.name, + old_dentry->d_name.len, + 0, &bh, + &link_address, &link_off, &link_fnode_block, 1); + if (unlikely(IS_ERR(fnode))) { + r = spadfs_free_blocks_metadata(fs, new_fxblk, 1U << fs->sectors_per_disk_block_bits); + if (unlikely(r)) + goto unlock_ret_r; +add_fnode_error: + if (likely(fnode == ERR_PTR(-ENOSPC)) && !synced) + goto unlock_do_sync; + r = PTR_ERR(fnode); + goto unlock_ret_r; + } + if (unlikely(fnode == NEED_SYNC)) { + r = spadfs_free_blocks_metadata(fs, new_fxblk, 1U << fs->sectors_per_disk_block_bits); + if (unlikely(r)) + goto unlock_ret_r; + goto unlock_do_sync; + } + BUG_ON(fnode == NEED_WLOCK); + + make_fixed_fnode_reference(fnode, new_fxblk); + fnode->namelen = old_dentry->d_name.len; + spadfs_set_name(fs, FNODE_NAME(fnode), + (const char *)old_dentry->d_name.name, + old_dentry->d_name.len); + do_fnode_block_checksum(fs, link_fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + + spadfs_remove_fnode_from_directory(spadfnode(old_dir), + spadfnode(file), &old_dentry->d_name); + + spadfnode(file)->fnode_block = new_fxblk; + spadfnode(file)->fnode_pos = FIXED_FNODE_BLOCK_FNODE0; + spadfnode(file)->spadfs_ino = + make_fixed_spadfs_ino_t(new_fxblk); + spadfs_set_parent_fnode(spadfnode(file), 0, 0); + remove_inode_hash(file); + __insert_inode_hash(file, spadfs_ino_t_2_ino_t(spadfnode(file)->spadfs_ino)); + spadfs_write_file(spadfnode(file), 0, NULL, NULL); + } + + fnode = spadfs_add_fnode_to_directory(spadfnode(new_dir), + (const char *)new_dentry->d_name.name, + new_dentry->d_name.len, + 0, &bh, + &link_address, &link_off, &link_fnode_block, 1); + if (unlikely(IS_ERR(fnode))) + goto add_fnode_error; + if (unlikely(fnode == NEED_SYNC)) + goto unlock_do_sync; + BUG_ON(fnode == NEED_WLOCK); + + make_fixed_fnode_reference(fnode, spadfnode(file)->fnode_block); + fnode->namelen = new_dentry->d_name.len; + spadfs_set_name(fs, FNODE_NAME(fnode), + (const char *)new_dentry->d_name.name, + new_dentry->d_name.len); + do_fnode_block_checksum(fs, link_fnode_block); + end_concurrent_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + + /* if (!S_ISDIR(inode(file)->i_mode)) mutex_lock(&spadfnode(new_file)->file_lock); we're already in fs write lock */ + spadfnode(file)->spadfs_nlink++; + spadfs_set_nlink(file); + r = spadfs_refile_fixed_fnode(spadfnode(file), NULL, 0); + if (unlikely(r)) { + spadfnode(file)->spadfs_nlink--; + spadfs_set_nlink(file); + } else { + atomic_inc(&file->i_count); + spadfs_update_directory_times(new_dir); + d_instantiate(new_dentry, file); + } + /* if (!S_ISDIR(inode(file)->i_mode)) mutex_unlock(&spadfnode(new_file)->file_lock); */ + up_write_sync_lock(fs); + return r; + +unlock_do_sync: + up_write_sync_lock(fs); + synced = 1; + r = spadfs_commit(fs); + if (unlikely(r)) + return r; + goto lock_and_again; + +unlock_ret_r: + up_write_sync_lock(fs); + return r; +} + +static void set_deleted_file(SPADFNODE *file) +{ + if (unlikely(is_fnode_fixed(file))) + spadfs_free_blocks_metadata(file->fs, file->fnode_block, 1U << file->fs->sectors_per_disk_block_bits); + + file->fnode_block = 0; + file->fnode_pos = 0; + spadfs_set_parent_fnode(file, 0, 0); + file->spadfs_ino = spadfs_ino_t_deleted; + remove_inode_hash(inode(file)); + __insert_inode_hash(inode(file), spadfs_ino_t_2_ino_t(spadfs_ino_t_deleted)); + file->spadfs_nlink = 0; + clear_nlink(inode(file)); +} + +int spadfs_unlink_unlocked(SPADFNODE *dir, struct dentry *dentry) +{ + SPADFNODE *file = spadfnode(dentry->d_inode); + int r; + + r = spadfs_remove_fnode_from_directory(dir, file, &dentry->d_name); + if (unlikely(r)) + return r; + + if (unlikely(file->spadfs_nlink != 1) && + likely(!S_ISDIR(inode(file)->i_mode))) { + file->spadfs_nlink--; + spadfs_set_nlink(inode(file)); + r = spadfs_refile_fixed_fnode(file, NULL, 0); + return r; + } + if (likely(!S_ISDIR(inode(file)->i_mode))) + spadfs_create_memory_extents(file); + + set_deleted_file(file); + return 0; +} + +static int spadfs_unlink(struct inode *dir, struct dentry *dentry) +{ + sync_lock_decl + int r; + SPADFNODE *file = spadfnode(dentry->d_inode); + +#ifdef SPADFS_QUOTA + dquot_initialize(dir); +#endif + + down_read_sync_lock(file->fs); + mutex_lock(&file->file_lock); + r = spadfs_unlink_unlocked(spadfnode(dir), dentry); + mutex_unlock(&file->file_lock); + if (!r) + spadfs_update_directory_times(dir); + up_read_sync_lock(file->fs); + + return r; +} + +static int spadfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + sync_lock_decl + SPADFNODE *file = spadfnode(dentry->d_inode); + int r; + +#ifdef SPADFS_QUOTA + dquot_initialize(dir); + dquot_initialize(inode(file)); +#endif + + down_read_sync_lock(file->fs); + + r = spadfs_check_directory_empty(file); + if (unlikely(r)) + goto unlock_ret_r; + + r = spadfs_remove_directory(file); + if (unlikely(r)) + goto unlock_ret_r; + + r = spadfs_unlink_unlocked(spadfnode(dir), dentry); + if (!r) + spadfs_update_directory_times(dir); + +unlock_ret_r: + up_read_sync_lock(file->fs); + return r; +} + +static int spadfs_rename( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) + struct mnt_idmap *ns, +#endif + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0) + , unsigned flags +#endif + ) +{ + /*sync_lock_decl*/ + SPADFS *fs = spadfnode(old_dir)->fs; + int r; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0) + if (unlikely(flags & ~RENAME_NOREPLACE)) + return -EINVAL; +#endif + +#ifdef SPADFS_QUOTA + dquot_initialize(old_dir); + dquot_initialize(new_dir); + if (new_dentry->d_inode) + dquot_initialize(new_dentry->d_inode); +#endif + + down_write_sync_lock(fs); + + r = spadfs_move_fnode_to_directory(spadfnode(old_dir), + &old_dentry->d_name, spadfnode(old_dentry->d_inode), + spadfnode(new_dir), &new_dentry->d_name, new_dentry, + NULL, 0); + + if (!r) { + spadfs_update_directory_times(old_dir); + spadfs_update_directory_times(new_dir); + } + + up_write_sync_lock(fs); + + return r; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) +static int spadfs_dir_setattr(struct dentry *dentry, struct iattr *iattr) +#else +static int spadfs_dir_setattr(struct mnt_idmap *ns, struct dentry *dentry, struct iattr *iattr) +#endif +{ + sync_lock_decl + struct inode *inode; + int r; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) + r = spadfs_setattr_common(dentry, iattr); +#else + r = spadfs_setattr_common(ns, dentry, iattr); +#endif + if (unlikely(r)) + return r; + + inode = dentry->d_inode; + + down_read_sync_lock(spadfnode(inode)->fs); + spadfs_update_ea(inode); + r = spadfs_write_directory(spadfnode(inode)); + up_read_sync_lock(spadfnode(inode)->fs); + + return r; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35) +static int spadfs_dir_fsync(struct file *file, struct dentry *dentry, + int datasync) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0) +static int spadfs_dir_fsync(struct file *file, int datasync) +#else +static int spadfs_dir_fsync(struct file *file, loff_t start, loff_t end, int datasync) +#endif +{ + struct inode *i = file_inode(file); + return spadfs_commit(spadfnode(i)->fs); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21) +const +#endif +struct inode_operations spadfs_dir_iops = { + .lookup = spadfs_lookup, + .create = spadfs_create, + .mkdir = spadfs_mkdir, + .mknod = spadfs_mknod, + .symlink = spadfs_symlink, + .link = spadfs_link, + .unlink = spadfs_unlink, + .rmdir = spadfs_rmdir, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0) || LINUX_VERSION_CODE >= KERNEL_VERSION(4,9,0) + .rename = spadfs_rename, +#else + .rename2 = spadfs_rename, +#endif + .setattr = spadfs_dir_setattr, + .getattr = spadfs_getattr, +#ifdef SPADFS_XATTR +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,9,0) + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .removexattr = generic_removexattr, +#endif + .listxattr = spadfs_listxattr, +#endif +}; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17) +const +#endif +struct file_operations spadfs_dir_fops = { + .llseek = generic_file_llseek, + .read = generic_read_dir, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0) + .readdir = spadfs_readdir, +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) + .iterate = spadfs_readdir, +#else + .iterate_shared = spadfs_readdir, +#endif + .fsync = spadfs_dir_fsync, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,11) + .unlocked_ioctl = spadfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = spadfs_compat_ioctl, +#endif +#endif +}; + diff --git a/fs/spadfs/spadfs.h b/fs/spadfs/spadfs.h new file mode 100644 index 000000000..fc9563554 --- /dev/null +++ b/fs/spadfs/spadfs.h @@ -0,0 +1,940 @@ +#ifndef __SPADFS_H +#define __SPADFS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,11,0) +#include +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,8,0) +#include +#endif + +#include "endian.h" + +#include "linux-compat.h" + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,23) || TEST_RHEL_VERSION(5,3) +#include +#endif + +#ifdef SPADFS_LITTLE_ENDIAN +#define SPAD2CPU16 le16_to_cpu +#define CPU2SPAD16 cpu_to_le16 +#define SPAD2CPU32 le32_to_cpu +#define CPU2SPAD32 cpu_to_le32 +#define SPAD2CPU64 le64_to_cpu +#define CPU2SPAD64 cpu_to_le64 +#define SPAD2CPU16_LV le16_to_cpup +#define CPU2SPAD16_LV(p, v) (*(p) = cpu_to_le16(v)) +#define SPAD2CPU32_LV le32_to_cpup +#define CPU2SPAD32_LV(p, v) (*(p) = cpu_to_le32(v)) +#define SPAD2CPU64_LV le64_to_cpup +#define CPU2SPAD64_LV(p, v) (*(p) = cpu_to_le64(v)) +#elif defined(SPADFS_BIG_ENDIAN) +#define SPAD2CPU16 be16_to_cpu +#define CPU2SPAD16 cpu_to_be16 +#define SPAD2CPU32 be32_to_cpu +#define CPU2SPAD32 cpu_to_be32 +#define SPAD2CPU64 be64_to_cpu +#define CPU2SPAD64 cpu_to_be64 +#define SPAD2CPU16_LV be16_to_cpup +#define CPU2SPAD16_LV(p, v) (*(p) = cpu_to_be16(v)) +#define SPAD2CPU32_LV be32_to_cpup +#define CPU2SPAD32_LV(p, v) (*(p) = cpu_to_be32(v)) +#define SPAD2CPU64_LV be64_to_cpup +#define CPU2SPAD64_LV(p, v) (*(p) = cpu_to_be64(v)) +#endif +#define SPAD2CPU16_CONST SPAD2CPU16 +#define CPU2SPAD16_CONST CPU2SPAD16 +#define SPAD2CPU32_CONST SPAD2CPU32 +#define CPU2SPAD32_CONST CPU2SPAD32 +#define SPAD2CPU64_CONST SPAD2CPU64 +#define CPU2SPAD64_CONST CPU2SPAD64 + +#define __make64(lo, hi) (((u64)(hi) << 32) | (u32)(lo)) +#define __finline__ inline + +#include "struct.h" + +/* #define CHECK_BUFFER_LEAKS */ +/* #define CHECK_BUFFER_WRITES */ +/* #define CHECK_BUFFER_WRITES_RANDOMIZE */ +/* #define CHECK_ALLOCMEM */ + +#define SPADFS_RESURRECT +#define SPADFS_OPTIMIZE_FSYNC + +#if !defined(CONFIG_SMP) || (defined(CONFIG_DEBUG_LOCK_ALLOC) && NR_CPUS > MAX_LOCKDEP_SUBCLASSES) +#define SIMPLE_SYNC_LOCK +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17) +#define SPADFS_DIRECT_IO +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24) +#define SPADFS_DO_PREFETCH +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,33) +#define SPADFS_XATTR +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,38) && defined(CONFIG_QUOTA) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) +#define SPADFS_QUOTA 3 +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) +#define SPADFS_QUOTA 2 +#else +#define SPADFS_QUOTA 1 +#endif +#endif + +#if defined(FITRIM) +#define SPADFS_FSTRIM +#endif + +/*#define SPADFS_META_PREALLOC*/ + +/* Allocate in other zones if the group has 1/2 avg free. Must be power of 2 */ +#define SPADFS_AVG_FREE_DIVIDE_OTHERZONE 2 + +#define SPADFS_DEFAULT_PREALLOC_PART 8 +#define SPADFS_DEFAULT_MIN_PREALLOC 4096 +#define SPADFS_DEFAULT_MAX_PREALLOC 1048576 +#define SPADFS_DEFAULT_METADATA_PREFETCH 524288 +#define SPADFS_DEFAULT_MAX_XFER_SIZE 1048576 +#define SPADFS_DEFAULT_SYNC_TIME (120 * HZ) + +#define SPADFS_INO_BATCH 65536 +#if BITS_PER_LONG == 32 +#define SPADFS_INODE_HASH_SIZE 32768 +#else +#define SPADFS_INODE_HASH_SIZE 262144 +#endif + +typedef struct __spadfnode SPADFNODE; +typedef struct __spadfs SPADFS; + +#define EFSERROR EUCLEAN + +#define MEMORY_SUPER_MAGIC ('S' | ('P' << 8) | ('A' << 16) | ('D' << 24)) + +static inline void spadfs_cond_resched(void) +{ +#ifndef CONFIG_PREEMPT + cond_resched(); +#endif +} + +/* + * Internal and external inode numbers: + * SpadFS doesn't have limit on number of files, but Linux has only 32-bit + * inode numbers. So the inode numbers can't be used to determine files. + * + * We use 64-bit value spadfs_ino_t to denote a file. This value has + * embedded sector number and position within sector. The value should be + * created with make_spadfs_ino_t and its parts can be extracted with + * spadfs_ino_t_sec and spadfs_ino_t_pos. Position must be >= 24, <= 448 + * and dividable by 8, as is specified by disk layout. + * + * Fixed fnodes have position "0" and sector number is the number of fixed + * fnode block. The are created with make_fixed_spadfs_ino_t and tested + * with is_pos_fixed or is_fnode_fixed. + * + * spadfs_ino_t_deleted is the special value that is assigned to + * deleted-but-open files. They have no on-disk representation. + * + * To keep userspace happy, we must manufacture 32-bit ino_t value somehow. + * Use the fact that no two fnodes can be less than 64-bytes apart --- so + * use top 3 bits of position and bottom 29 bits of sector number. + * spadfs_ino_t_2_ino_t does this. If the disk has at most 256GiB, ino_t + * will be collision-free, otherwise collisions are possible but unlikely. + * + * Kernels starting with 2.6.19 can return 64-bit ino, so + * spadfs_ino_t_2_ino64_t returns 64-bit value on them. + */ + +typedef u64 spadfs_ino_t; + +#define make_spadfs_ino_t(sec, pos) ((sec) | ((u64)(pos) << 55)) +#define make_fixed_spadfs_ino_t(sec) make_spadfs_ino_t(sec, 0) +#define spadfs_ino_t_sec(ino) ((sector_t)(ino) & (sector_t)(((u64)1 << 48) - 1)) +#define spadfs_ino_t_pos(ino) ((unsigned)((ino) >> 55)) + +#define spadfs_ino_t_2_ino_t(ino) ((((u32)((ino) >> 61) | ((u32)(ino) << 3)) & (SPADFS_INO_INITIAL_REGION - 1)) | SPADFS_INO_INITIAL_REGION) +#define spadfs_ino_t_2_ino64_t(ino) ((u64)spadfs_ino_t_2_ino_t(ino) | (((ino) << 5) & (((u64)1 << 53) - ((u64)1 << 32)))) + +#define spadfs_ino_t_deleted ((spadfs_ino_t)1) +#define is_pos_fixed(pos) (!(pos)) + +#define is_fnode_fixed(fnode) (is_pos_fixed(spadfs_ino_t_pos((fnode)->spadfs_ino))) + +#define is_deleted_file(fnode) (!(fnode)->fnode_block) + +/* + * Encoding directory position into 64-bit off_t. Ideally, the directory + * position would need 32+48+3 bits (hash+block in chain+position), really we + * strip block in chain to 29 bits --- for directories that have more than 2^29 + * blocks in hash-collision chain, readdir won't work. + * + * Rotate the bits somehow so that if old application accesses directory with + * only 32-bit offset, it most time works (16 bits --- hash, 3 bits --- + * position, 13 bits --- chain). + * + * Special directory offsets --- used to return entries "." and ".." that do not + * really exist in the directory. Readdir can use macros is_dir_off_t_special, + * dir_off_t_special_n and make_special_dir_off_t to implement simple state + * machine. + */ + +#define make_dir_off_t(hash, order, pos) (((hash) & 0xffff) | ((((pos) + (0x40 - SIZEOF_FNODE_BLOCK)) & 0x1c0) << (16 - 6)) | (((order) & 0x1fff) << (16 + 3)) | (((u64)((hash) & 0xffff0000) << 16) | ((u64)((order) & 0x1fffe000) << (32 + 3)))) +#define dir_off_t_hash(off) (((u32)(off) & 0xffff) | ((u32)((off) >> 16) & 0xffff0000)) +#define dir_off_t_order(off) ((((u32)(off) >> (16 + 3)) & 0x1fff) | ((u32)((off) >> (32 + 3)) & 0x1fffe000)) +#define dir_off_t_pos(off) ((((u32)(off) >> (16 - 6)) & 0x1c0) - (0x40 - SIZEOF_FNODE_BLOCK)) + +#define is_dir_off_t_special(off) (!((u32)(off) & (0x1c0 << (16 - 6)))) +#define dir_off_t_special_n(off) ((u32)(off)) +#define make_special_dir_off_t(n) (n) + +#define MAX_CHAIN_LENGTH ((1 << 29) - 1) + +/* + * One mapped apage. Really these structures form array of + * [ pagesize / 2 * blocksize ] APAGE_MAPs. + * Buffer head, entry is bh->b_data; + */ + +typedef struct { + struct aentry *entry; /* [ blocksize / 16 ] */ + struct buffer_head *bh; +} APAGE_MAP; /* [ pagesize / 2 / blocksize ] */ + +/* + * Describes a zone. A total 3 zones exist --- metadata, small files, large + * files. + * grp_start of next zone must be grp_start+grp_n of previous zone, grp_start of + * the first zone must be 0 --- a lot of code depends on it. + */ + +struct spadfszone { + sector_t freespace; + unsigned grp_start; + unsigned grp_n; +}; + +/* + * Describes a group. + */ + +struct spadfs_group_info { + sector_t freespace; + struct spadfszone *zone; +}; + +struct extent_cache { + sector_t physical_sector; + sector_t logical_sector; + unsigned long n_sectors; +} ____cacheline_aligned_in_smp; + +struct alloc_reservation { + struct rb_node rb_node; + sector_t start; + unsigned len; +}; + +#define FNODE_EA_INLINE ((unsigned)sizeof(struct ea_unx)) + +struct __spadfnode { + SPADFS *fs; + loff_t disk_size; /* Size of file on disk (including prealloc) */ + /* If commit_sequence == fs->commit_sequence, then crash_disk_size is + * the valid file size in case of crash. */ + loff_t crash_disk_size; + u64 commit_sequence; + loff_t mmu_private; + spadfs_ino_t spadfs_ino; + spadfs_ino_t stable_ino; + + /* Copy of entries in fnode on disk */ + sector_t blk1; + sector_t blk2; + unsigned blk1_n; + unsigned blk2_n; + sector_t root; + + /* This fnode's and parent fnode's block and position. For fixed fnodes, + * parent values are 0. */ + sector_t fnode_block; + sector_t parent_fnode_block; + unsigned fnode_pos; + unsigned parent_fnode_pos; + + /* List of all inodes */ + struct hlist_node inode_list; + + struct inode vfs_inode; + + /* Only for files. Directories are reasonably protected by the kernel */ + /* FIXME: can be removed too? will i_mutex do the right thing? It should + be checked */ + struct mutex file_lock; + + unsigned long target_blocks; + int target_blocks_exact; + int dont_truncate_prealloc; + + /* Real number of links, see comment on spadfs_set_nlink */ + u64 spadfs_nlink; + + struct alloc_reservation res; + + struct list_head clear_entry; + sector_t clear_position; + +#if defined(SPADFS_QUOTA) && SPADFS_QUOTA >= 2 + struct dquot *i_dquot[MAXQUOTAS]; +#endif + + /* Pointer to UNX attribute within "ea" array to speed-up access. + * Or NULL if there is no UNX attribute. */ + struct ea_unx *ea_unx; + + /* Extended attributes */ + unsigned ea_size; + u8 *ea; + u8 ea_inline[FNODE_EA_INLINE] __attribute__((__aligned__(FNODE_EA_ALIGN))); + + /* Cache for one extent --- physical block, logical block, number of + * blocks */ + u64 extent_cache_seq; + struct extent_cache extent_cache[0]; +}; + +#define PREALLOC_TIMEOUT HZ +#define PREALLOC_DISCARD_TIMEOUT (15 * HZ) +#define PREALLOC_THRESHOLD 8 + +struct prealloc_state { + unsigned long last_alloc; + unsigned n_allocations; + unsigned allocations_size; + sector_t sector; + unsigned n_sectors; + pid_t pgrp; +}; + +struct __spadfs { + u32 *cct; /* Crash count table --- an array of 65536 32-bit values */ + u16 cc; /* Current crash count */ + s32 txc; /* cct[cc] */ + s32 a_txc; /* txc/cc pair selecting one of two apage indices */ + u16 a_cc; + +/* Values selected with mkspadfs. "_bits" suffix means log2 of the value */ + unsigned char sectors_per_buffer_bits; + unsigned char sectors_per_disk_block_bits; + unsigned char sectors_per_fnodepage_bits; + unsigned char sectors_per_page_bits; + unsigned char pagesize_bits; + unsigned char dnode_hash_bits; + unsigned char sectors_per_cluster_bits; + unsigned char sectors_per_group_bits; + unsigned apage_size; + unsigned dnode_data_size; + unsigned dnode_page_sectors; + unsigned n_apage_mappings; + u32 cluster_threshold; + +/* Mount options */ + uid_t uid; + gid_t gid; + umode_t mode; + s8 prealloc_part_bits; + unsigned prealloc_part; + unsigned min_prealloc; + unsigned max_prealloc; + unsigned metadata_prefetch; + unsigned xfer_size; + +/* Values read from superblock */ + sector_t txb_sec; + sector_t apage_index0_sec; + sector_t apage_index1_sec; + sector_t cct_sec; + sector_t root_sec; + sector_t size; + sector_t freespace; + sector_t reserve_sectors; + sector_t group_mask; + unsigned max_allocation; + unsigned max_freed_run; + u32 flags_compat_fsck; + u32 flags_compat_rw; + u32 flags_compat_ro; + u32 flags_compat_none; + + unsigned n_apages; /* Total apages */ + unsigned n_active_apages; /* Used apages */ + struct apage_index_entry *apage_index; + struct apage_info *apage_info; + unsigned cached_apage_buffers; + unsigned mapped_apage_buffers; + struct list_head apage_lru; + + unsigned n_groups; /* Total groups */ + struct spadfs_group_info *group_info; /* Group descriptors */ + struct spadfszone zones[3]; /* Zone descriptors */ + + APAGE_MAP *tmp_map; + struct mutex alloc_lock; +#ifdef SPADFS_META_PREALLOC + struct prealloc_state meta_prealloc; +#endif + struct prealloc_state small_prealloc; + struct rb_root alloc_mem; + sector_t alloc_mem_sectors; + + struct rb_root alloc_reservations; + + int need_background_sync; + + struct hlist_head *inode_list;/* List of all inodes */ + struct mutex inode_list_lock;/* List's lock */ + +#ifdef SPADFS_QUOTA + /* Prevent quota file extension concurrently with commit */ + struct mutex quota_alloc_lock; +#endif + + struct super_block *s; /* VFS part of superblock */ + + u64 stable_ino; /* Next free inode number */ + spinlock_t stable_ino_lock; /* lock for "ino" */ + + struct workqueue_struct *spadfs_syncer; /* Syncer thread */ + unsigned long spadfs_sync_time; /* Sync time */ +#ifndef NEW_WORKQUEUE + struct work_struct spadfs_sync_work; +#else + struct delayed_work spadfs_sync_work; +#endif + + unsigned char mount_flags; + unsigned char split_happened; + + u32 txflags; /* Error flags */ + u32 txflags_new; /* New error flags (if txflags_new & ~txflags, + error flags need to be written) */ + spinlock_t txflags_new_lock; + + u64 commit_sequence; + + struct mutex trim_lock; + sector_t trim_start; + unsigned trim_len; + + spinlock_t clear_lock; + struct list_head clear_list; + + int buffer_size; + +#ifdef CHECK_BUFFER_LEAKS + struct mutex buffer_track_lock; + struct hlist_head buffer_list; + unsigned long buffer_oom_events; +#endif + +#ifdef SIMPLE_SYNC_LOCK + struct rw_semaphore sync_lock; +#else + struct rw_semaphore *sync_locks[0]; +#endif +}; + +#define MOUNT_FLAGS_CHECKSUMS_OVERRIDE 0x01 +#define MOUNT_FLAGS_CHECKSUMS 0x02 +#define MOUNT_FLAGS_CLEAR_DIRTY_ON_UNMOUNT 0x04 +#define MOUNT_FLAGS_64BIT_INO 0x08 +#define MOUNT_FLAGS_64BIT_INO_FORCE 0x10 +#define MOUNT_FLAGS_USRQUOTA 0x20 +#define MOUNT_FLAGS_GRPQUOTA 0x40 + +static inline u32 spadfs_magic(SPADFS *fs, sector_t block, u32 magic) +{ + if (likely(fs->flags_compat_none & FLAG_COMPAT_NONE_DYNAMIC_MAGIC)) + magic ^= block; + return magic; +} + +struct apage_mapping { + APAGE_MAP *map; + struct list_head list; +}; + +struct apage_info { + struct apage_mapping mapping[2]; +}; + +#define spadfs(s) ((SPADFS *)((s)->s_fs_info)) +#define spadfnode(i) list_entry(i, SPADFNODE, vfs_inode) +#define inode(i) (&(i)->vfs_inode) + +#ifndef CONFIG_SMP +#define spadfs_nr_cpus 1 +#define spadfs_unlocked_extent_cache 1 +#define spadfs_extent_cache_size 1 +#else +extern unsigned spadfs_nr_cpus; +extern int spadfs_unlocked_extent_cache; +extern unsigned spadfs_extent_cache_size; +#endif + +#ifdef SIMPLE_SYNC_LOCK +#define init_sync_lock(fs, fail) init_rwsem(&(fs)->sync_lock) +#define done_sync_lock(fs) do { } while (0) +#define down_read_sync_lock(fs) (down_read(&(fs)->sync_lock)) +#define up_read_sync_lock(fs) (up_read(&(fs)->sync_lock)) +#define down_write_sync_lock(fs) (down_write(&(fs)->sync_lock)) +#define up_write_sync_lock(fs) (up_write(&(fs)->sync_lock)) +#define sync_lock_decl +/* Linux doesn't allow us to differentiate between lock for read and write */ +#define assert_write_sync_lock(fs) BUG_ON(!rwsem_is_locked(&(fs)->sync_lock)) +#else +void spadfs_do_down_read_sync_lock(SPADFS *fs, unsigned *cpu); +void spadfs_do_up_read_sync_lock(SPADFS *fs, unsigned cpu); +void spadfs_do_down_write_sync_lock(SPADFS *fs); +void spadfs_do_up_write_sync_lock(SPADFS *fs); +#define init_sync_lock(fs, fail) do { \ + if (spadfs_do_init_sync_lock(fs)) { \ + fail \ + } \ + } while (0) +#define done_sync_lock(fs) spadfs_do_done_sync_lock(fs) +#define down_read_sync_lock(fs) spadfs_do_down_read_sync_lock(fs, &cpu) +#define up_read_sync_lock(fs) spadfs_do_up_read_sync_lock(fs, cpu) +#define down_write_sync_lock(fs) spadfs_do_down_write_sync_lock(fs) +#define up_write_sync_lock(fs) spadfs_do_up_write_sync_lock(fs) +#define sync_lock_decl unsigned cpu; +/* Linux doesn't allow us to differentiate between lock for read and write */ +#define assert_write_sync_lock(fs) BUG_ON(!rwsem_is_locked((fs)->sync_locks[0])) +#endif + +static inline int CC_VALID(SPADFS *fs, u16 *cc, s32 *txc) +{ + return (s32)( + (u32)SPAD2CPU32_LV(&fs->cct[SPAD2CPU16_LV(cc)]) - + (u32)SPAD2CPU32_LV((u32 *)txc) + ) >= 0; +} + +static inline int CC_CURRENT(SPADFS *fs, u16 *cc, s32 *txc) +{ + return !( + (SPAD2CPU16_LV(cc) ^ fs->cc) | + (u32)((SPAD2CPU32_LV((u32 *)txc) ^ fs->txc) & 0x7fffffff) + ); +} + +static inline void CC_SET_CURRENT(SPADFS *fs, u16 *cc, s32 *txc) +{ + CPU2SPAD32_LV(txc, (~( + (u32)SPAD2CPU32_LV(&fs->cct[SPAD2CPU16_LV(cc)]) - + (u32)SPAD2CPU32_LV((u32 *)txc) + ) & 0x80000000U) | fs->txc); + CPU2SPAD16_LV(cc, fs->cc); +} + +static inline void CC_SET_CURRENT_INVALID(SPADFS *fs, u16 *cc, s32 *txc) +{ + CPU2SPAD32_LV(txc, fs->txc ^ 0x80000000U); + CPU2SPAD16_LV(cc, fs->cc); +} + +/* + * Use this if you don't want partially modified buffer be written to disk. + */ + +static void set_need_background_sync(SPADFS *fs) +{ + if (unlikely(!READ_ONCE(fs->need_background_sync))) + WRITE_ONCE(fs->need_background_sync, 1); +} + +static inline void start_atomic_buffer_modify(SPADFS *fs, + struct buffer_head *bh) +{ + clear_buffer_dirty(bh); + wait_on_buffer(bh); +} + +static inline void end_atomic_buffer_modify(SPADFS *fs, struct buffer_head *bh) +{ + set_need_background_sync(fs); + mark_buffer_dirty(bh); +} + +/* + * The same functionality, but can be called simultaneously for the same buffer. + * Need to serialize. + */ + +static inline void start_concurrent_atomic_buffer_modify(SPADFS *fs, + struct buffer_head *bh) +{ + lock_buffer(bh); +} + +static inline void end_concurrent_atomic_buffer_modify(SPADFS *fs, + struct buffer_head *bh) +{ + set_need_background_sync(fs); + unlock_buffer(bh); + mark_buffer_dirty(bh); +} + +static inline void end_concurrent_atomic_buffer_modify_nodirty(SPADFS *fs, + struct buffer_head *bh) +{ + unlock_buffer(bh); +} + +/* + * The same as previous, but read-only access. Used when checking checksums. + */ + +static inline void start_concurrent_atomic_buffer_read(SPADFS *fs, + struct buffer_head *bh) +{ + lock_buffer(bh); +} + +static inline void end_concurrent_atomic_buffer_read(SPADFS *fs, + struct buffer_head *bh) +{ + unlock_buffer(bh); +} + +#define check_checksums(fs) \ + (!((fs)->flags_compat_fsck & FLAG_COMPAT_FSCK_NO_CHECKSUMS)) +#define make_checksums(fs) \ + (!((fs)->flags_compat_fsck & FLAG_COMPAT_FSCK_NO_CHECKSUMS)) +#define directory_size(fs) \ + ((fs)->flags_compat_ro & FLAG_COMPAT_RO_DIRECTORY_SIZES) + +static inline loff_t spadfs_roundup_blocksize(SPADFS *fs, loff_t size) +{ + unsigned mask = (512U << fs->sectors_per_disk_block_bits) - 1; + return (size + mask) & ~(loff_t)mask; +} + +/* + * SpadFS allows more than 2^32 hardlinks to a file, however i_nlink is only + * 32-bit. So keep internal 64-bit value "spadfs_nlink" and external value + * i_nlink exposed to stat(). + */ + +static inline void spadfs_set_nlink(struct inode *inode) +{ + unsigned nlink = spadfnode(inode)->spadfs_nlink; + if (unlikely(nlink != spadfnode(inode)->spadfs_nlink)) nlink = -1; + set_nlink(inode, nlink); +} + +/* alloc.c */ + +#define ALLOC_METADATA 0x00010000 +#define ALLOC_SMALL_FILE 0x00020000 +#define ALLOC_BIG_FILE 0x00040000 +#define ALLOC_PARTIAL_AT_GOAL 0x00080000 +#define ALLOC_NEW_GROUP_HINT 0x00100000 +#ifdef SPADFS_RESURRECT +#define ALLOC_RESURRECT 0x00200000 +#endif + +#define ALLOC_FREE_FROM 0x80000000 + +#define ALLOC_MASK_MASK 0x0000FFFF +#define ALLOC_MASK_1 0x00000001 +#define ALLOC_MASK(flags) (((flags) / ALLOC_MASK_1) & ALLOC_MASK_MASK) + +struct alloc { + sector_t sector; + sector_t top; + unsigned n_sectors; + unsigned extra_sectors; + unsigned flags; + struct alloc_reservation *reservation; +}; + +void spadfs_discard_reservation(SPADFS *fs, struct alloc_reservation *res); +void spadfs_prune_cached_apage_buffers(SPADFS *fs); +int spadfs_count_free_space(SPADFS *fs); +sector_t spadfs_get_freespace(SPADFS *fs); +int spadfs_alloc_blocks(SPADFS *fs, struct alloc *al); +int spadfs_free_blocks_unlocked(SPADFS *fs, sector_t start, + sector_t n_sectors); + +int spadfs_free_blocks_metadata(SPADFS *f, sector_t start, + sector_t n_sectors); +void spadfs_prealloc_discard_unlocked(SPADFS *fs, struct prealloc_state *prealloc); +void spadfs_reclaim_max_allocation(SPADFS *fs); + +/* allocmem.c */ + +struct allocmem { + struct rb_node rb_node; + sector_t start; + sector_t len; +}; + +void spadfs_allocmem_init(SPADFS *fs); +void spadfs_allocmem_done(SPADFS *fs); +int spadfs_allocmem_find(SPADFS *fs, sector_t off, sector_t n_sec, sector_t *next_change); +int spadfs_allocmem_add(SPADFS *fs, sector_t off, sector_t n_sec); +void spadfs_allocmem_delete(SPADFS *fs, sector_t off, sector_t n_sec); +#ifdef SPADFS_FSTRIM +int spadfs_trim_fs(SPADFS *fs, u64 start, u64 end, u64 minlen, sector_t *result); +#endif +#ifdef CHECK_ALLOCMEM +int spadfs_allocmem_unit_test(SPADFS *fs); +#endif + +/* buffer.c */ + +static inline void *spadfs_buffer_data(SPADFS *fs, sector_t secno, struct buffer_head *bh) +{ + return bh->b_data + (((unsigned long)secno & ((1U << fs->sectors_per_buffer_bits) - 1)) << 9); +} + +void *spadfs_read_sector(SPADFS *fs, sector_t secno, struct buffer_head **bhp, unsigned ahead, const char *msg); +void *spadfs_get_new_sector(SPADFS *fs, sector_t secno, struct buffer_head **bhp, const char *msg); +#if 0 +void spadfs_prefetch_sector(SPADFS *fs, sector_t secno, unsigned ahead, const char *msg); +#endif +void spadfs_discard_buffers(SPADFS *fs, sector_t start, sector_t n_sectors); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) +static inline int spadfs_sync_dirty_buffer(struct buffer_head *bh) +{ + sync_dirty_buffer(bh); + return 0; +} +#else +#define spadfs_sync_dirty_buffer sync_dirty_buffer +#endif + +#ifndef CHECK_BUFFER_LEAKS +#define spadfs_drop_reference(fs, buf) do { } while (0) +#ifndef CHECK_BUFFER_WRITES +#define spadfs_brelse(fs, buf) __brelse(buf) +#else +#define spadfs_brelse(fs, buf) do { spadfs_sync_dirty_buffer(buf); __brelse(buf); } while (0) +#endif +#define spadfs_buffer_leaks_init(fs) do { } while (0) +#define spadfs_buffer_leaks_done(fs) do { } while (0) +#else +void spadfs_drop_reference(SPADFS *fs, struct buffer_head *bh); +void spadfs_brelse(SPADFS *fs, struct buffer_head *bh); +void spadfs_buffer_leaks_init(SPADFS *fs); +void spadfs_buffer_leaks_done(SPADFS *fs); +#endif + +/* bufstruc.c */ + +struct txblock *spadfs_read_tx_block(SPADFS *fs, struct buffer_head **bhp, const char *msg); +#define SRFB_FNODE 1 +#define SRFB_FIXED_FNODE 2 +#define SRFB_DNODE 4 +struct fnode_block *spadfs_read_fnode_block(SPADFS *fs, sector_t secno, struct buffer_head **bhp, int struct_type, const char *msg); +struct anode *spadfs_read_anode(SPADFS *fs, sector_t secno, struct buffer_head **bhp, unsigned *vx, int read_lock, const char *msg); + +/* dir.c */ + +#define HINT_META 0 +#define HINT_SMALL 1 +#define HINT_BIG 2 + +sector_t spadfs_alloc_hint(SPADFNODE *f, int hint); +void spadfs_set_new_hint(SPADFNODE *f, struct alloc *al); +void spadfs_get_dir_hint(SPADFNODE *f, u16 *small, u16 *big); +int spadfs_write_file(SPADFNODE *f, int datasync, int *optimized, struct buffer_head **bhp); +int spadfs_write_directory(SPADFNODE *f); + +int spadfs_refile_fixed_fnode(SPADFNODE *file, u8 *new_ea, unsigned new_ea_size); +int spadfs_refile_fnode(SPADFNODE *dir, struct qstr *qstr, SPADFNODE *file, u8 *new_ea, unsigned new_ea_size); + +/* file.c */ + +sector_t spadfs_size_2_sectors(SPADFS *fs, loff_t size); + +void spadfs_create_memory_extents(SPADFNODE *f); +void spadfs_clear_last_block(struct inode *i); +void spadfs_truncate(struct inode *i); +void spadfs_delete_file_content(SPADFNODE *f); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) +int spadfs_file_setattr(struct dentry *dentry, struct iattr *iattr); +#else +int spadfs_file_setattr(struct mnt_idmap *ns, struct dentry *dentry, struct iattr *iattr); +#endif + +extern +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21) +const +#endif +struct inode_operations spadfs_file_iops; +extern +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17) +const +#endif +struct file_operations spadfs_file_fops; +extern +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) +const +#endif +struct address_space_operations spadfs_file_aops; + +#ifdef SPADFS_QUOTA +ssize_t spadfs_quota_read(struct super_block *s, int type, + char *data, size_t len, loff_t off); +ssize_t spadfs_quota_write(struct super_block *s, int type, + const char *data, size_t len, loff_t off); +#if SPADFS_QUOTA >= 2 +struct dquot **spadfs_quota_get(struct inode *inode); +#endif +#endif + +/* inode.c */ + +struct fnode_ea *spadfs_get_ea(SPADFNODE *f, u32 what, u32 mask); +void spadfs_find_ea_unx(SPADFNODE *f); +void spadfs_validate_stable_ino(u64 *result, u64 ino); +u64 spadfs_expose_inode_number(SPADFS *fs, u64 ino); +int spadfs_get_fixed_fnode_pos(SPADFS *fs, struct fnode_block *fnode_block, sector_t secno, unsigned *pos); +struct inode *spadfs_iget(struct super_block *s, spadfs_ino_t ino, sector_t parent_block, unsigned parent_pos); +struct inode *spadfs_new_inode(struct inode *dir, umode_t mode, time_t ctime, void *ea, unsigned ea_size, sector_t fnode_block, unsigned fnode_pos, sector_t dir_root, u64 stable_ino); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) +void spadfs_delete_inode(struct inode *i); +#else +void spadfs_evict_inode(struct inode *i); +#endif +u64 spadfs_get_user_inode_number(struct inode *inode); +void spadfs_get_initial_attributes(struct inode *dir, umode_t *mode, uid_t *uid, gid_t *gid); +void spadfs_init_unx_attribute(struct inode *dir, struct ea_unx *ea, umode_t mode, u64 stable_ino); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) +int spadfs_setattr_common(struct dentry *dentry, struct iattr *iattr); +#else +int spadfs_setattr_common(struct mnt_idmap *ns, struct dentry *dentry, struct iattr *iattr); +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0) +int spadfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) +int spadfs_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned query_flags); +#else +int spadfs_getattr(struct mnt_idmap *ns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned query_flags); +#endif +void spadfs_update_ea(struct inode *inode); +void spadfs_set_parent_fnode(SPADFNODE *f, sector_t sec, unsigned pos); +void spadfs_move_parent_dir_ptr(SPADFS *fs, sector_t src_sec, unsigned src_pos, sector_t dst_sec, unsigned dst_pos); +void spadfs_move_fnode_ptr(SPADFS *fs, sector_t src_sec, unsigned src_pos, sector_t dst_sec, unsigned dst_pos, int is_dir); + +/* ioctl.c */ + +long spadfs_ioctl(struct file *file, unsigned cmd, unsigned long arg); +#ifdef CONFIG_COMPAT +long spadfs_compat_ioctl(struct file *file, unsigned cmd, unsigned long arg); +#endif + +/* link.c */ + +extern +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21) +const +#endif +struct inode_operations spadfs_symlink_iops; + +/* name.c */ + +int spadfs_compare_names(SPADFS *fs, const char *n1, unsigned l1, const char *n2, unsigned l2); +void spadfs_set_name(SPADFS *fs, char *dest, const char *src, unsigned len); + +extern +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30) +const +#endif +struct dentry_operations spadfs_dops; + +static inline void spadfs_set_dentry_operations(SPADFS *fs, struct dentry *dentry) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) + if (unlikely(!(fs->flags_compat_none & FLAG_COMPAT_NONE_UNIX_NAMES))) + dentry->d_op = &spadfs_dops; +#endif +} + +/* namei.c */ + +int spadfs_unlink_unlocked(SPADFNODE *dir, struct dentry *dentry); + +extern +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21) +const +#endif +struct inode_operations spadfs_dir_iops; +extern +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17) +const +#endif +struct file_operations spadfs_dir_fops; + +/* super.c */ + +extern spadfs_cache_t *spadfs_extent_cachep; + +int spadfs_ea_alloc(SPADFNODE *f, unsigned ea_size); +static inline int spadfs_ea_resize(SPADFNODE *f, unsigned ea_size) +{ + if (likely(ea_size <= FNODE_EA_INLINE)) + return 0; + return spadfs_ea_alloc(f, ea_size); +} + +__cold void __attribute__ ((__format__ (__printf__, 3, 4))) spadfs_error_(SPADFS *fs, unsigned flags, const char *msg, ...); +#define spadfs_error(fs, flags, msg, ...) spadfs_error_(fs, flags, KERN_ERR "spadfs: " msg "\n", ## __VA_ARGS__) + +int spadfs_stop_cycles(SPADFS *fs, sector_t key, sector_t (*c)[2], const char *msg); +int spadfs_issue_flush(SPADFS *fs); +void spadfs_tx_block_checksum(struct txblock *tb); +int spadfs_new_stable_ino(SPADFS *fs, u64 *stable_ino); +int spadfs_commit_unlocked(SPADFS *fs); +int spadfs_commit(SPADFS *fs); + +/* xattr.c */ + +ssize_t spadfs_listxattr(struct dentry *dentry, char *list, size_t list_size); +extern const struct xattr_handler *spadfs_xattr_handlers[]; + +#endif diff --git a/fs/spadfs/struct.h b/fs/spadfs/struct.h new file mode 100644 index 000000000..d214d304a --- /dev/null +++ b/fs/spadfs/struct.h @@ -0,0 +1,965 @@ +#ifndef _SPADFS_COMMON_STRUCT_H +#define _SPADFS_COMMON_STRUCT_H + +/* ******************* + * DATA STRUCTURES * + ******************* +disk: (align) +superblock block +txblock block +apage_index[0] block +apage_index[1] block +apages page (alloc apages from last to first) +cc-table block +root_fnode block +root_fnode_directory block + +memory: (alloc method) +fs (superblock/txblock) in FS struct +apage_index grab_contig_area(DATA|PHYSCONTIG) +apage_index_info grab_contig_area(DATA) + (total&contig free blocks in each apage + checksumless flag) +cc-table grab_contig_area(DATA|PHYSCONTIG) +*/ + +#define CHECKSUM_BASE ((__u8)'C') + +#define TXC_INVALID(txc) ((__s32)((__u32)(txc) - 0x80000000U)) + +#define MAKE_D_OFF(l,h) (__make64(SPAD2CPU32_LV(&l), SPAD2CPU16_LV(&h))) +#define MAKE_PART_0(l) (CPU2SPAD32((__u32)(l))) +#define MAKE_PART_1(l) (CPU2SPAD16((__u16)((l) >> 31 >> 1))) + +#define MAKE_D_OFF3(l,h,hh) (__make64(SPAD2CPU16_LV(&l) | \ + ((__u32)SPAD2CPU16_LV(&h) << 16),\ + SPAD2CPU16_LV(&hh))) +#define MAKE_PART_30(l) (CPU2SPAD16((__u16)(l))) +#define MAKE_PART_31(l) (CPU2SPAD16((__u16)((__u32)(l) >> 16))) +#define MAKE_PART_32(l) (CPU2SPAD16((__u16)((l) >> 31 >> 1))) + + +#define BOOTLOADER_SECTORS 2 + +#define USERBLOCK_SECTOR 16383 + +#define USERBLOCK_MAGIC 0x004C4255 + +#define USERFLAGS_NEED_FSCK 0x01 +#define USERFLAGS_REBUILD_APAGES 0x02 +#define USERFLAGS_RESET_CC 0x04 +#define USERFLAGS_EXTEND 0x08 +#define USERFLAGS_SET_RESERVE 0x10 +#define USERFLAGS_ZERO_FILE_TAILS 0x20 + +struct userblock { + __u32 magic; + __u8 checksum; + __u8 userflags; + __u8 pad[2]; + __u64 reserve; +}; + +#define SUPERBLOCK_SECTOR 16384 + +#define SUPERBLOCK_SIGNATURE "FS SPAD\000" + +#define SPADFS_VERSION_MAJOR 1 +#define SPADFS_VERSION_MIDDLE 1 +#define SPADFS_VERSION_MINOR 18 + +#define CCT_SIZE (65536*4) /* do not change it */ +#define SSIZE_BITS 9 /* --- "" --- */ + +struct superblock { + __u8 signature[8]; + + __u64 byte_sex; /* 0x01234567890ABCDEF */ + + __u8 version_major; /* must be EXACLY SAME */ + __u8 version_middle; /* don't care */ + __u8 version_minor; /* don't care */ + __u8 checksum; + + __u8 sectors_per_block_bits; /* 0 -- block 512 bytes */ + __u8 sectors_per_fnodepage_bits;/* 4 -- fnode 8192 bytes */ + __u8 sectors_per_page_bits; /* 6 -- page 32768 bytes */ + __u8 pad1; + + __u8 sectors_per_cluster_bits; /* 6 -- that means pad files to 32768 */ + __u8 sectors_per_group_bits; + __u16 cluster_threshold; /* 4 -- that is pad files to 131072 */ + __u16 small_file_group; + __u16 large_file_group; + + __u64 size; + __u64 reserve_sectors; + + __u64 txblock; + __u64 apage_index[2]; + __u64 cct; + __u64 root; + __u64 pad2; + __u32 flags_compat_fsck; + __u32 flags_compat_rw; + __u32 flags_compat_ro; + __u32 flags_compat_none; +}; + +#define FLAG_COMPAT_FSCK_NO_CHECKSUMS 0x00000001 +#define FLAG_COMPAT_RO_DIRECTORY_SIZES 0x00000001 +#define FLAG_COMPAT_NONE_UNIX_NAMES 0x00000001 +#define FLAG_COMPAT_NONE_DYNAMIC_MAGIC 0x00000002 +#define FLAG_COMPAT_NONE_EXTRA_SPACE 0x00000004 + +#define MAX_SECTORS_PER_BLOCK_BITS 7 +#define MAX_SECTORS_PER_PAGE_BITS 8 +#define MAX_ALLOC_MASK_BITS 16 +#define MIN_SIZE (SUPERBLOCK_SECTOR + 2048) +#define MAX_SIZE ((__u64)0x00010000 << 32) +#define SIZE_MASK (MAX_SIZE - 1) + +#define TXBLOCK_MAGIC 0x4C425854 + +#define TXFLAGS_DIRTY 0x01 +#define TXFLAGS_IO_READ_ERROR 0x02 +#define TXFLAGS_IO_WRITE_ERROR 0x04 +#define TXFLAGS_FS_ERROR 0x08 +#define TXFLAGS_CHECKSUM_ERROR 0x10 +#define TXFLAGS_EA_ERROR 0x20 + +struct txblock { + __u32 magic; + __u8 checksum; + __u8 pad; + __u16 cc; + + __u16 pad2; + __u16 a_cc; /* if CC_VALID(a_cc, a_txc), apage_index[0] is valid, */ + __s32 a_txc; /* otherwise apage_index[1] */ + __u32 txflags; + __u32 pad3; + __u64 ino; +}; + +#define SPADFS_INO_INITIAL_REGION 0x40000000 +#define SPADFS_INO_ROOT 1 + +/********* + * APAGE * + *********/ + +#define APAGE_INDEX_ENTRY_SIZE (sizeof(struct apage_index_entry)) + +struct apage_index_entry { + __u64 apage; + __u64 end_sector; /* 0 means free apage */ +}; + +#define APAGE_HEAD_SIZE (sizeof(struct apage_head)) + +#define APAGE_MAGIC 0x5041 + +#define APAGE_SIZE_BITS 0x07 +#define APAGE_SIZE_BITS_SHIFT 0 +#define APAGE_BLOCKSIZE_BITS 0x38 +#define APAGE_BLOCKSIZE_BITS_SHIFT 3 +#define APAGE_CHECKSUM_VALID 0x40 +#define APAGE_BITMAP 0x80 + +#define APAGE_SIZE(flags) ((1 << SSIZE_BITS) << \ + (((flags) & APAGE_SIZE_BITS) >> APAGE_SIZE_BITS_SHIFT)) +#define APAGE_SECTORS_PER_BLOCK_BITS(flags) \ + (((flags) & APAGE_BLOCKSIZE_BITS) >> APAGE_BLOCKSIZE_BITS_SHIFT) + +struct apage_subhead { + union { + struct { /* for list */ + __u8 flags; + __u8 checksum; + __u16 freelist; + __u16 last; + __u16 first; + } l; + struct { /* for bitmap */ + __u8 flags; + __u8 checksum; + __u16 start1; + __u32 start0; + } b; + } u; +}; + +struct apage_head { /* must: sizeof(apage_head) == sizeof(aentry) */ + __u16 magic; /* these 3 entries are only in the first half */ + __u16 cc; /* if CC_VALID(cc, txc), first half of page is valid, */ + __s32 txc; /* otherwise second page */ + + /* following entries are used on both halves of page */ + struct apage_subhead s; +}; + +struct aentry { + __u64 start; + __u32 len; /* 0 -- entry is free (and is on freelist) */ + __u16 prev; /* 0 -- head is previous (this entry is first) */ + __u16 next; /* 0 -- head is next (this entry is last) */ +}; + +#define APTR_ALIGN(x) ((__u16)(x) & ~(sizeof(struct aentry) - 1)) +/* APTR_ALIGN must not return number < 0 */ +#define APTR_INVALID(x, asize) (unlikely((x) & (sizeof(struct aentry) - 1)) ||\ + unlikely((x) >= (asize))) + +#define BITMAP_SIZE(d) (((d) - sizeof(struct apage_head)) * 8) + +#define BITMAP_OFFSET(a, o) (unlikely((((o) - (MAKE_D_OFF((a)->s.u.b.start0, (a)->s.u.b.start1))) & ~(__u64)0xffffffffu) != 0) ? (unsigned)-1 : ((unsigned)(o) - (unsigned)SPAD2CPU32_LV(&(a)->s.u.b.start0)) >> APAGE_SECTORS_PER_BLOCK_BITS((a)->s.u.b.flags)) +#define BITMAP_LEN(a, l) ((l) >> APAGE_SECTORS_PER_BLOCK_BITS((a)->s.u.b.flags)) + +#define BITMAP_CLEAR(a, o) \ +do { \ + __u32 *bp_ = ((__u32 *)((__u8 *)(a) + sizeof(struct apage_head)) + \ + ((o) >> 5)); \ + CPU2SPAD32_LV(bp_, SPAD2CPU32_LV(bp_) & ~(1 << ((o) & 31))); \ +} while (0) + +#define BITMAP_SET(a, o) \ +do { \ + __u32 *bp_ = ((__u32 *)((__u8 *)(a) + sizeof(struct apage_head)) + \ + ((o) >> 5)); \ + CPU2SPAD32_LV(bp_, SPAD2CPU32_LV(bp_) | (1 << ((o) & 31))); \ +} while (0) + + /* must return 0 or 1 */ +#define BITMAP_TEST(a, o) ((SPAD2CPU32_LV((__u32 *)((__u8 *)(a) + \ + sizeof(struct apage_head)) + \ + ((o) >> 5)) >> ((o) & 31)) & 1) + +#define BITMAP_TEST_32_FULL(a, o) (*((__u32 *)((__u8 *)(a) + \ + sizeof(struct apage_head)) + \ + ((o) >> 5)) == \ + CPU2SPAD32_CONST(0xffffffffu)) + +/* + * if apage contains <= CONV_APAGE_SECTORS sectors, convert it to bitmap + * instead of splitting it + */ +#define CONV_APAGE_SECTORS(page_bits, block_bits) \ + ((((1 << (page_bits)) / 2 - APAGE_HEAD_SIZE) * 8) << \ + ((block_bits) - SSIZE_BITS)) + +/* minimum sectors in an apage */ +#define MIN_APAGE_SECTORS(page_bits, block_bits) \ + (CONV_APAGE_SECTORS(page_bits, block_bits) / 2) + +/* leave 1/8 of apage free when rebuilding to avoid immediate split */ +#define FSCK_APAGE_FREE_PERCENTAGE 8 + +/* number of apages for filesystem of given size */ +#if !(defined(__linux__) && defined(__KERNEL__)) +#define N_APAGES(size, page_bits, block_bits) \ + (((size) + MIN_APAGE_SECTORS(page_bits, block_bits) - 1 - 1) / \ + MIN_APAGE_SECTORS(page_bits, block_bits) + 1) +#else +static __finline__ __u64 N_APAGES(__u64 size, int page_bits, int block_bits) +{ + int a = 1; + int mas = MIN_APAGE_SECTORS(page_bits, block_bits); + __u64 as = mas; + while (as + 1 < size) { + a++; + as += mas; + } + return a + 1; +} +#endif + +#define MAX_APAGES 0x7ffffffe + +/* number of sectors in each apage index */ +#define APAGE_INDEX_SECTORS(n_apages, blocksize) \ + ((((__u64)(n_apages) * APAGE_INDEX_ENTRY_SIZE + (blocksize) - 1) &\ + ~(__u64)((blocksize) - 1)) >> SSIZE_BITS) + +/********* + * FNODE * + *********/ + +/* + * 0x1F8 would be sufficient, but test more bits so that + * fsck can resync better on corrupted directories + */ +#define FNODE_NEXT_SIZE 0x7FF8 +#define FNODE_NEXT_FREE 0x0001 +/* + * fnode is free if: (next & FNODE_NEXT_FREE) && CC_VALID(cc, txc) + * fnode can be allocated if: is_free && !CC_CURRENT(cc, txc) + * alloc: cc,txc = CURRENT ^ 0x80000000 + * FNODE_NEXT_FREE = 1 + * write at pos 1. + * fnode can be updated if !is_free + * update: if (!CC_CURRENT) CC_SET_CURRENT, FNODE_NEXT_FREE = 0; + * write at pos !CC_VALID(cc, txc) + * fnode can be deleted if !is_free + * delete: if (CC_CURRENT && FNODE_NEXT_FREE) cc, txc = 0; + * else { + * if (CC_VALID) copy data from pos 0 -> to pos 1; + * cc, txc = CURRENT; + * FNODE_NEXT_FREE = 1; + * } + * free fnodes can be joined if both can be allocated. + */ + +#define FNODE_FLAGS_DIR 0x01 +#define FNODE_FLAGS_HARDLINK 0x02 + +#define FNODE_HEAD_SIZE 8 + +struct fnode { + __u16 next; /* size of fnode */ + __u16 cc; + __s32 txc; + __u64 size[2]; /* 0 if CC_VALID(cc, txc), 1 if !CC_VALID(cc, txc) */ + __u32 ctime; + __u32 mtime; + +#define MAX_DIRECT_BLKS(blksize) (0x10000U - (blksize)) + + __u32 run10; + __u16 run11; + __u16 run1n; + __u32 run20; + __u16 run21; + __u16 run2n; + __u32 anode0; + __u16 anode1; + /* + * for directories: + * if CC_VALID(cc, txc), run10/run11 is the root page; + * otherwise run20/run21 is the root page + * for hardlinks: anode0/anode1 is pointer to the fixed_fnode_block + */ + __u8 flags; + + __u8 namelen; +}; + +/* + * this might be smaller than actual filesystem blocksize -- it is beacuse + * block device guarantees atomic write of 512 bytes but not more. + */ +#define FNODE_BLOCK_SIZE 512 +/* + * sizeof(struct fnode_block) + */ +#define SIZEOF_FNODE_BLOCK (6 * 4) + + +#define FNODE_MAX_SIZE (FNODE_BLOCK_SIZE - SIZEOF_FNODE_BLOCK)/* 488 */ +#define FNODE_EMPTY_MIN_SIZE 8 + +#define FNODE_NAME_POS (sizeof(struct fnode)) +#define FNODE_NAME(fnode) ((char *)(fnode) + FNODE_NAME_POS) + +#define MAX_NAME_LEN 255 + +#define INVALID_FILENAME_CHARACTER(pos, c, unix_names) \ + (unlikely(!(c)) || \ + unlikely(((c) == '/')) || \ + (!(unix_names) && \ + ((unlikely((c) >= 'a') && unlikely((c) <= 'z')) || \ + unlikely((c) == ':') || \ + unlikely((unsigned char)(c) < ' ') || \ + (unlikely((c) == '^') && !(pos))))) + +#define FNODE_EA_POS(name_len) (((FNODE_NAME_POS + (name_len)) + 7) & ~7) +#define FNODE_MAX_EA_SIZE (FNODE_MAX_SIZE - sizeof(struct fnode) - 256) + /* 176 */ + +#define FNODE_SIZE(name_len, ea_size) \ + ((FNODE_EA_POS(name_len) + (ea_size) + 7) & ~7) + +struct fnode_ea { + __u32 magic; +}; + +#define FNODE_EA_MAGIC_MASK 0x00FFFFFF +#define FNODE_EA_SIZE_SHIFT 24 +#define FNODE_EA_SIZE_MASK_1 0xFF +#define FNODE_EA_SIZE_ADD (7 + 4) +#define FNODE_EA_SIZE_MASK_2 0xFFFFFFF8 +#define FNODE_EA_ALIGN 8 +#define FNODE_EA_DO_ALIGN(n) (((n) + FNODE_EA_ALIGN - 1) & \ + ~(FNODE_EA_ALIGN - 1)) + +#define GET_EA_ERROR ((void *)1) + +static __finline__ struct fnode_ea *GET_EA(struct fnode_ea *ea, unsigned ea_size, + __u32 what, __u32 mask) +{ + while (unlikely(ea_size)) { + unsigned rec_size = + (((SPAD2CPU32_LV(&ea->magic) >> FNODE_EA_SIZE_SHIFT) & + FNODE_EA_SIZE_MASK_1) + FNODE_EA_SIZE_ADD) & + FNODE_EA_SIZE_MASK_2; + if (unlikely(rec_size > ea_size)) + return GET_EA_ERROR; + if ((SPAD2CPU32_LV(&ea->magic) & mask) == (what & mask)) + return ea; + ea = (struct fnode_ea *)((char *)ea + rec_size); + ea_size -= rec_size; + } + return NULL; +} + +static __finline__ int RESIZE_EA(__u8 *ea_pool, unsigned *ea_pool_size, + struct fnode_ea *ea, unsigned new_size) +{ + unsigned old_size = (SPAD2CPU32_LV(&ea->magic) >> FNODE_EA_SIZE_SHIFT) & + FNODE_EA_SIZE_MASK_1; + unsigned asize_1 = + (old_size + FNODE_EA_SIZE_ADD) & FNODE_EA_SIZE_MASK_2; + unsigned asize_2 = + (new_size + FNODE_EA_SIZE_ADD) & FNODE_EA_SIZE_MASK_2; + if (unlikely(*ea_pool_size - asize_1 + asize_2 > FNODE_MAX_EA_SIZE)) + return -ENOSPC; + memmove((__u8 *)ea + asize_2, (__u8 *)ea + asize_1, + (ea_pool + *ea_pool_size) - ((__u8 *)ea + asize_1)); + memset((__u8 *)ea + sizeof(struct fnode_ea) + new_size, 0, + asize_2 - (new_size + sizeof(struct fnode_ea))); + *ea_pool_size = *ea_pool_size - asize_1 + asize_2; + CPU2SPAD32_LV(&ea->magic, (SPAD2CPU32_LV(&ea->magic) & + ~(FNODE_EA_SIZE_MASK_1 << FNODE_EA_SIZE_SHIFT)) | + (new_size << FNODE_EA_SIZE_SHIFT)); + return 0; +} + +static __finline__ void REMOVE_EA(__u8 *ea_pool, unsigned *ea_pool_size, + struct fnode_ea *ea) +{ + unsigned old_size = (SPAD2CPU32_LV(&ea->magic) >> FNODE_EA_SIZE_SHIFT) & + FNODE_EA_SIZE_MASK_1; + unsigned asize_1 = + (old_size + FNODE_EA_SIZE_ADD) & FNODE_EA_SIZE_MASK_2; + memmove((__u8 *)ea, (__u8 *)ea + asize_1, + (ea_pool + *ea_pool_size) - ((__u8 *)ea + asize_1)); + *ea_pool_size = *ea_pool_size - asize_1; +} + +#define EA_UNX_MAGIC_MASK ((__u32)~(0x08 << 24)) +#define EA_UNX_MAGIC ((0x00584E55 | \ + ((sizeof(struct ea_unx) - sizeof(struct fnode_ea)) << 24))) +#define EA_UNX_MAGIC_OLD ((0x00584E55 | \ + ((sizeof(struct ea_unx) - sizeof(struct fnode_ea)) << 24)) & \ + EA_UNX_MAGIC_MASK) +#define EA_SYMLINK_MAGIC_MASK FNODE_EA_MAGIC_MASK +#define EA_SYMLINK_MAGIC 0x004D5953 +#define EA_RDEV_MAGIC_MASK ((__u32)~0) +#define EA_RDEV_MAGIC (0x00524E55 | \ + ((sizeof(struct ea_rdev) - sizeof(struct fnode_ea)) << \ + FNODE_EA_SIZE_SHIFT)) +#define EA_XATTR_MAGIC_MASK FNODE_EA_MAGIC_MASK +#define EA_XATTR_MAGIC 0x00544158 + +struct ea_unx { + __u32 magic; + __u16 mode; + __u16 flags; /* not used now */ + __u32 uid; + __u32 gid; + __u32 prealloc[2]; + __u64 ino; +}; + +struct ea_rdev { + __u32 magic; + __u32 pad; + __u64 dev; +}; + +/* +struct ea_xattr { + __u32 magic; + __u8 type; + __u8 namelen; + __u8 valuelen; + __u8 name[]; + __u8 value[]; +}; +*/ + +/* mode in ea_unx is compatible with Linux flags */ +#define LINUX_S_IFMT 0170000 +#define LINUX_S_IFSOCK 0140000 +/* LINUX_S_IFLNK must not be set, symlinks are recognised by "SYM" attribute */ +#define LINUX_S_IFLNK 0120000 +#define LINUX_S_IFREG 0100000 +#define LINUX_S_IFBLK 0060000 +#define LINUX_S_IFDIR 0040000 +#define LINUX_S_IFCHR 0020000 +#define LINUX_S_IFIFO 0010000 + +#define LINUX_S_ISUID 0004000 +#define LINUX_S_ISGID 0002000 +#define LINUX_S_ISVTX 0001000 + +#define LINUX_S_IRWXU 00700 +#define LINUX_S_IRUSR 00400 +#define LINUX_S_IWUSR 00200 +#define LINUX_S_IXUSR 00100 + +#define LINUX_S_IRWXG 00070 +#define LINUX_S_IRGRP 00040 +#define LINUX_S_IWGRP 00020 +#define LINUX_S_IXGRP 00010 + +#define LINUX_S_IRWXO 00007 +#define LINUX_S_IROTH 00004 +#define LINUX_S_IWOTH 00002 +#define LINUX_S_IXOTH 00001 + +#define LINUX_S_ISLNK(m) (((m) & LINUX_S_IFMT) == LINUX_S_IFLNK) +#define LINUX_S_ISREG(m) (((m) & LINUX_S_IFMT) == LINUX_S_IFREG) +#define LINUX_S_ISDIR(m) (((m) & LINUX_S_IFMT) == LINUX_S_IFDIR) +#define LINUX_S_ISCHR(m) (((m) & LINUX_S_IFMT) == LINUX_S_IFCHR) +#define LINUX_S_ISBLK(m) (((m) & LINUX_S_IFMT) == LINUX_S_IFBLK) +#define LINUX_S_ISFIFO(m) (((m) & LINUX_S_IFMT) == LINUX_S_IFIFO) +#define LINUX_S_ISSOCK(m) (((m) & LINUX_S_IFMT) == LINUX_S_IFSOCK) + + +#define SPADFS_XATTR_END 0x00 +#define SPADFS_XATTR_USER 0x55 +#define SPADFS_XATTR_TRUSTED 0x54 +#define SPADFS_XATTR_SECURITY 0x53 +#define SPADFS_XATTR_ACL_ACCESS 0x41 +#define SPADFS_XATTR_ACL_DEFAULT 0x44 + +#define GET_XAT_ERROR ((void *)1) + +#define GET_XAT_TYPE_NAME 0 +#define GET_XAT_TYPE 1 +#define GET_XAT_ALL 2 + +static __finline__ __u8 *GET_XAT(__u8 *xat, unsigned xat_size, int mode, int type, + const char *name, unsigned namelen) +{ + while (xat_size) { + unsigned this_size; + + if (unlikely(xat_size < 3)) + return GET_XAT_ERROR; + + if (unlikely(!xat[2])) + return GET_XAT_ERROR; + + this_size = 3 + xat[1] + xat[2]; + + if (unlikely(this_size > xat_size)) + return GET_XAT_ERROR; + + if (unlikely(mode == GET_XAT_ALL)) + return xat; + if (type == xat[0]) { + if (mode == GET_XAT_TYPE || + (xat[1] == namelen && + !memcmp(xat + 3, name, namelen))) + return xat; + } + + xat += this_size; + xat_size -= this_size; + } + return NULL; +} + + +#define FILE_SECTORS(bl_size, cl_size, cl_thresh, size, result) \ +do { \ + unsigned bsize_ = (bl_size); \ + if ((size) >= (cl_thresh)) bsize_ = (cl_size); \ + result = (((size) + bsize_ - 1) & ~(__u64)(bsize_ - 1)) >> SSIZE_BITS;\ +} while (0) + +#define FNODE_BLOCK_MAGIC 0x444F4E46 +#define FNODE_BLOCK_CHECKSUM_VALID 0x01 +#define FNODE_BLOCK_LAST 0x02 +#define FNODE_BLOCK_FIRST 0x04 + +struct fnode_block { + __u32 magic; + __u8 flags; + __u8 checksum; + __u16 prev1; /* 0 if first */ + __u32 prev0; /* --- "" --- */ + __s32 txc; + __u16 cc; + __u16 next1; /* valid if CC_VALID(cc, txc) */ + __u32 next0; /* ---------- "" ------------ */ + struct fnode fnodes[1]; +}; + +#define FIXED_FNODE_BLOCK_SIZE 512 +#define FIXED_FNODE_BLOCK_MAGIC 0x4E465846 +#define FIXED_FNODE_BLOCK_CHECKSUM_VALID FNODE_BLOCK_CHECKSUM_VALID + +struct fixed_fnode_block { + __u32 magic; + /* + * flags & checksum must match struct fnode_block + * --- the same checksum routine is used. + */ + __u8 flags; + __u8 checksum; + __u16 cc; + __s32 txc; + __u16 hint_small; + __u16 hint_large; + /* &fnode0-&nlink0 must be equal to &fnode1-&nlink1 */ + __u64 nlink0; /* if CC_VALID(cc, txc) */ + __u8 fnode0[FNODE_MAX_SIZE - 256]; /* 232 */ + __u64 reserved0; + __u64 nlink1; /* otherwise */ + __u8 fnode1[FNODE_MAX_SIZE - 256]; /* 232 */ + __u64 reserved1; +}; + +#define FIXED_FNODE_BLOCK_FNODE0 \ + ((int)(long)&(((struct fixed_fnode_block *)NULL)->fnode0)) +#define FIXED_FNODE_BLOCK_NLINK0 \ + ((int)(long)&(((struct fixed_fnode_block *)NULL)->nlink0)) +#define FIXED_FNODE_BLOCK_FNODE1 \ + ((int)(long)&(((struct fixed_fnode_block *)NULL)->fnode1)) +#define FIXED_FNODE_BLOCK_NLINK1 \ + ((int)(long)&(((struct fixed_fnode_block *)NULL)->nlink1)) + +#define FIXED_FNODE_NLINK_PTR(fnode) \ + ((__u64 *)((char *)(fnode) - \ + (FIXED_FNODE_BLOCK_FNODE0 - FIXED_FNODE_BLOCK_NLINK0))) + +struct dnode_page_entry { + __u16 b0; + __u16 b1; + __u16 b2; +} +#if defined(__KERNEL__) || defined(__GNUC__) +__attribute__((packed)) +#endif +; + +#define DNODE_PAGE_ENTRY_SIZE sizeof(struct dnode_page_entry) +#define DNODE_PAGE_ENTRY_BITS 3 + +#define DNODE_PAGE_MAGIC 0x444F4E44 + +#define DNODE_CHECKSUM_VALID 0x01 + + /* used for fsck */ +#define DNODE_GFLAGS_PAGE_SIZE_BITS_MINUS_1 0x07 +#define DNODE_GFLAGS_PAGE_SIZE_BITS_MINUS_1_SHIFT 0 + +struct dnode_page { + __u32 magic; + __u8 flags[2]; + __u8 gflags; + __u8 pad; + __u64 up_dnode; /* 0 if top-level */ + __s32 txc; + __u16 cc; + __u8 checksum[2]; +}; + +#define DNODE_ENTRY_OFFSET sizeof(struct dnode_page) + +/* + * if (CC_VALID(cc, txc)) the first dnode version is valid + * else the second version is valid + */ + +typedef __u32 hash_t; +#define SPADFS_HASH_BITS 32 + +static __finline__ hash_t name_hash(const char *name) +{ + hash_t hash = 0; + while (*name) + hash = hash * 15 + (*name++ & 0xdf); + return hash; +} + +static __finline__ hash_t name_len_hash(const char *name, int len) +{ + hash_t hash = 0; + while (len--) + hash = hash * 15 + (*name++ & 0xdf); + return hash; +} + +/********* + * ANODE * + *********/ + +struct extent { + __u64 blk; + __u64 end_off; +}; + +#define ANODE_MAGIC 0x444F4E41 + +#define ANODE_CHECKSUM_VALID 0x01 +#define ANODE_ROOT 0x02 + +#define ANODE_ROOT_NAME(anode) ((char *) \ + &(anode)->x[ANODE_N_EXTENTS - 1]) +#define ANODE_ROOT_NAME_LEN (sizeof(struct extent)) + +#define ANODE_SIZE 512 +#define ANODE_N_EXTENTS 31 + +struct anode { + __u32 magic; + __u8 flags; + __u8 checksum; + __u8 valid_extents; + __u8 pad; + __u64 start_off; + struct extent x[ANODE_N_EXTENTS]; +}; + +static __finline__ int find_direct(int depth_now, int depth_total) +{ + if (likely(!depth_now)) + return 20; /* 20 -- direct; 10 -- indirect; 1 -- name */ + if (depth_now == depth_total) + return 31; + return 1; +} + +static __finline__ void update_depth(int *depth_now, int *depth_total, int off) +{ + if (likely(!*depth_now)) + *depth_total = off - 20 + 1; + (*depth_now)++; +} + +static __finline__ unsigned find_in_anode(struct anode *ano, __u64 lbn, unsigned h) +{ + unsigned l = 0; + h--; + if (unlikely(h >= ANODE_N_EXTENTS)) + h = 0; + while (l < h) { + unsigned d = (l + h) >> 1; + if (lbn >= SPAD2CPU64_LV(&ano->x[d].end_off)) + l = d + 1; + else + h = d; + } + return l; +} + +#define FILE_END_1_MAGIC (((__u64)0x4946 << 32) | 0x315F454C) +#define FILE_END_2_MAGIC (((__u64)0x4946 << 32) | 0x325F454C) + +struct file_end { + __u32 size; + __u8 sectors_per_block_bits; + __u8 sectors_per_cluster_bits; + __u16 magic1; + __u32 magic0; +}; + +struct file_end_2 { + __u32 run10; + __u16 run11; + __u16 run1n; + __u32 size; + __u8 sectors_per_block_bits; + __u8 sectors_per_cluster_bits; + __u16 magic1; + __u32 magic0; +}; + +static __finline__ int validate_range(__u64 fssize, unsigned blockmask, __u64 start, __u64 len) +{ + __u64 end; + if (unlikely(((unsigned)start | (unsigned)len) & blockmask)) + return 0; + end = start + len; + if (unlikely(end <= start)) + return 0; + if (unlikely(end > fssize)) + return 0; + if (unlikely(start <= SUPERBLOCK_SECTOR)) { + if (unlikely(end > USERBLOCK_SECTOR)) + return 0; + if (unlikely(start < BOOTLOADER_SECTORS)) + return 0; + } + return 1; +} + +static __finline__ const char *validate_super(struct superblock *super) +{ + unsigned blockmask; + __u64 groups; + __u64 n_apages; + __u64 apage_index_sectors; + if (unlikely(super->sectors_per_block_bits > + MAX_SECTORS_PER_BLOCK_BITS)) + return "invalid sectors_per_block_bits"; + if (unlikely(super->sectors_per_cluster_bits >= 31 - SSIZE_BITS)) + return "invalid sectors_per_cluster_bits"; + if (unlikely(super->sectors_per_cluster_bits > MAX_ALLOC_MASK_BITS)) + return "too large sectors_per_cluster_bits"; + if (unlikely(super->sectors_per_page_bits < 1)) + return "zero sectors_per_page_bits"; + if (unlikely(super->sectors_per_page_bits > MAX_SECTORS_PER_PAGE_BITS)) + return "invalid sectors_per_page_bits"; + if (unlikely(super->sectors_per_block_bits > + super->sectors_per_cluster_bits)) + return "sectors_per_block_bits larger than sectors_per_cluster_bits"; + if (unlikely(super->sectors_per_block_bits > + super->sectors_per_page_bits)) + return "sectors_per_block_bits larger than sectors_per_page_bits"; + if (unlikely(super->sectors_per_fnodepage_bits > + super->sectors_per_page_bits)) + return "sectors_per_fnodepage_bits larger than sectors_per_page_bits"; + if (unlikely(super->sectors_per_fnodepage_bits < + super->sectors_per_block_bits)) + return "sectors_per_block_bits larger than sectors_per_fnodepage_bits"; + if (unlikely((__u64)SPAD2CPU16_LV(&super->cluster_threshold) << + super->sectors_per_cluster_bits >= 1 << (31 - SSIZE_BITS))) + return "too large cluster threshold"; + if (unlikely(super->sectors_per_group_bits > 48)) + return "invalid sectors_per_group_bits"; + if (unlikely(super->sectors_per_group_bits < + super->sectors_per_cluster_bits)) + return "sectors_per_cluster_bits larger than sectors_per_group_bits"; + blockmask = (1 << super->sectors_per_block_bits) - 1; + if (unlikely(SPAD2CPU64_LV(&super->size) < MIN_SIZE)) + return "size too small"; + if (unlikely(SPAD2CPU64_LV(&super->size) > MAX_SIZE)) + return "size too large"; + if (unlikely((unsigned)SPAD2CPU64_LV(&super->size) & blockmask)) + return "size unaligned"; + groups = (SPAD2CPU64_LV(&super->size) + + ((__u64)1 << super->sectors_per_group_bits) - 1) >> + super->sectors_per_group_bits; + if (unlikely(groups > 0xffff)) + return "too many groups (group size too small)"; + if (unlikely(SPAD2CPU16_LV(&super->small_file_group) > groups)) + return "invalid small file group"; + if (unlikely(SPAD2CPU16_LV(&super->large_file_group) < + SPAD2CPU16_LV(&super->small_file_group)) || + unlikely(SPAD2CPU16_LV(&super->large_file_group) > groups)) + return "invalid large file group"; + if (unlikely(!validate_range(SPAD2CPU64_LV(&super->size), blockmask, SPAD2CPU64_LV(&super->txblock), blockmask + 1))) + return "txblock invalid"; + n_apages = N_APAGES(SPAD2CPU64_LV(&super->size), + super->sectors_per_page_bits + SSIZE_BITS, + super->sectors_per_block_bits + SSIZE_BITS); + if (unlikely(n_apages >= MAX_APAGES)) + return "too many apages"; + apage_index_sectors = APAGE_INDEX_SECTORS(n_apages, + 1 << SSIZE_BITS << super->sectors_per_block_bits); + if (unlikely(!validate_range(SPAD2CPU64_LV(&super->size), blockmask, SPAD2CPU64_LV(&super->apage_index[0]), apage_index_sectors))) + return "apage_index[0] invalid"; + if (unlikely(!validate_range(SPAD2CPU64_LV(&super->size), blockmask, SPAD2CPU64_LV(&super->apage_index[1]), apage_index_sectors))) + return "apage_index[1] invalid"; + if (unlikely(!validate_range(SPAD2CPU64_LV(&super->size), blockmask, SPAD2CPU64_LV(&super->cct), CCT_SIZE >> SSIZE_BITS))) + return "cct invalid"; + if (unlikely(!validate_range(SPAD2CPU64_LV(&super->size), blockmask, SPAD2CPU64_LV(&super->root), blockmask + 1))) + return "root invalid"; + return (char *)0; +} + +#define RESERVE_PERCENT_SMALL 1/50 +#define RESERVE_PERCENT_BIG 1/200 +#define RESERVE_LIMIT ((__u64)67108864/512) + +static __finline__ __u64 get_default_reserved(__u64 size) +{ + __u64 reserve = size * RESERVE_PERCENT_SMALL; + if (reserve > RESERVE_LIMIT) { + reserve = RESERVE_LIMIT; + if (reserve < size * RESERVE_PERCENT_BIG) + reserve = size * RESERVE_PERCENT_BIG; + } + return reserve; +} + +#define OPTIMAL_GROUPS 512 +#define MINIMAL_GROUP_SECTORS 131072 +#define MINIMAL_GROUPS 32 + +static __finline__ unsigned get_default_group_bits(__u64 size, unsigned cluster_bits) +{ + unsigned group_bits; + + for (group_bits = 1; + (size + ((__u64)1 << (group_bits - 1))) >> group_bits > + OPTIMAL_GROUPS || + (__u64)1 << group_bits < MINIMAL_GROUP_SECTORS; + group_bits++) ; + + while (group_bits && (size + ((__u64)1 << group_bits) - 1) >> group_bits + < MINIMAL_GROUPS) + group_bits--; + if (group_bits < cluster_bits) + group_bits = cluster_bits; + return group_bits; +} + +#define METADATA_PART 36 +#define SMALLFILE_PART 8 + +static __finline__ unsigned get_default_metadata_groups(unsigned group_bits, + unsigned groups) +{ + unsigned metadata_groups; + unsigned min_groups; + metadata_groups = groups / METADATA_PART; + min_groups = (SUPERBLOCK_SECTOR + (CCT_SIZE >> SSIZE_BITS)) >> + group_bits; + if (metadata_groups <= min_groups) + metadata_groups = min_groups + 1; + if (metadata_groups >= groups) + metadata_groups = 0; + return metadata_groups; +} + +static __finline__ unsigned get_default_smallfile_groups(unsigned group_bits, + unsigned groups, + unsigned metadata_groups) +{ + unsigned smallfile_groups; + groups -= metadata_groups; + smallfile_groups = groups / SMALLFILE_PART; + if (!smallfile_groups && groups >= 2) + smallfile_groups = 1; + return smallfile_groups; +} + +#ifndef __SPAD__ +static __finline__ unsigned char __byte_sum(void *__ptr, int __len) +{ + unsigned long __sum = 0; + void *__e = (char *)__ptr + __len; + barrier(); + do { + __sum ^= *(unsigned long *)__ptr; + __ptr = (char *)__ptr + sizeof(unsigned long); + } while (__ptr < __e); + __sum ^= __sum >> 31 >> 1; + __sum ^= __sum >> 16; + __sum ^= __sum >> 8; + barrier(); + return __sum; +} +#endif + +static __finline__ int spadfs_struct_check_correctness(void) +{ + return sizeof(struct dnode_page_entry) == 6; +} + +#endif diff --git a/fs/spadfs/super.c b/fs/spadfs/super.c new file mode 100644 index 000000000..9d155a304 --- /dev/null +++ b/fs/spadfs/super.c @@ -0,0 +1,1741 @@ +#include "spadfs.h" + +__cold static void spadfs_free_super(struct super_block *s); + +static +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21) +const +#endif +struct super_operations spadfs_sops; +static spadfs_cache_t *spadfs_inode_cachep = NULL; +static spadfs_cache_t *spadfs_ea_cachep = NULL; +spadfs_cache_t *spadfs_extent_cachep = NULL; + +/* Slab contructor */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) +static void fnode_ctor(void *fn_, spadfs_cache_t *cachep, unsigned long flags) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) +static void fnode_ctor(spadfs_cache_t *cachep, void *fn_) +#else +static void fnode_ctor(void *fn_) +#endif +{ + SPADFNODE *fn = fn_; +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + if (likely((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == + SLAB_CTOR_CONSTRUCTOR)) +#endif + { + inode_init_once(inode(fn)); + fn->res.len = 0; + INIT_LIST_HEAD(&fn->clear_entry); + fn->ea = fn->ea_inline; + if (spadfs_unlocked_extent_cache) + fn->extent_cache_seq = 0; + } +} + +/* + * Allocate/free inode. The list is walked and searched for + * parent_fnode_block/parent_fnode_pos, so we'd better initialize them. + */ + +static struct inode *spadfs_alloc_inode(struct super_block *s) +{ + SPADFS *fs = spadfs(s); + SPADFNODE *fn; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,18,0) + fn = alloc_inode_sb(s, spadfs_inode_cachep, GFP_NOFS); +#else + fn = kmem_cache_alloc(spadfs_inode_cachep, GFP_NOFS); +#endif + if (unlikely(!fn)) + return NULL; + + fn->fs = fs; + fn->parent_fnode_block = 0; + fn->parent_fnode_pos = 0; + fn->target_blocks = 0; + fn->target_blocks_exact = 0; + fn->dont_truncate_prealloc = 0; + fn->ea_size = 0; + + mutex_init(&fn->file_lock); + +#if defined(SPADFS_QUOTA) && SPADFS_QUOTA >= 2 + memset(&fn->i_dquot, 0, sizeof fn->i_dquot); +#endif + + return inode(fn); +} + +static void spadfs_destroy_inode(struct inode *inode) +{ + SPADFNODE *fn = spadfnode(inode); + + if (fn->parent_fnode_block) + spadfs_set_parent_fnode(fn, 0, 0); + + if (unlikely(fn->ea != fn->ea_inline)) { + kmem_cache_free(spadfs_ea_cachep, fn->ea); + fn->ea = fn->ea_inline; + } + + mutex_destroy(&fn->file_lock); + + kmem_cache_free(spadfs_inode_cachep, fn); +} + +int spadfs_ea_alloc(SPADFNODE *f, unsigned ea_size) +{ + u8 *ea; + BUG_ON(ea_size > FNODE_MAX_EA_SIZE); + if (likely(f->ea != f->ea_inline)) + return 0; + ea = kmem_cache_alloc(spadfs_ea_cachep, GFP_NOFS); + if (unlikely(!ea)) + return -ENOMEM; + memcpy(ea, f->ea_inline, f->ea_size); + f->ea = ea; + spadfs_find_ea_unx(f); + return 0; +} + +static void set_prealloc_part(SPADFS *fs, unsigned prealloc_part) +{ + fs->prealloc_part = prealloc_part; + if (!(fs->prealloc_part & (fs->prealloc_part - 1))) + fs->prealloc_part_bits = ffs(fs->prealloc_part) - 1; + else + fs->prealloc_part_bits = -1; +} + +enum { + Opt_help, Opt_uid, Opt_gid, Opt_umask, + Opt_prealloc_part, Opt_prealloc_min, Opt_prealloc_max, + Opt_xfer_size, Opt_buffer_size, Opt_prefetch, Opt_sync_time, + Opt_no_checksums, Opt_checksums, + Opt_ino64, + Opt_usrquota, Opt_grpquota, + Opt_err, +}; + +static match_table_t tokens = { + {Opt_help, "help"}, + {Opt_uid, "uid=%u"}, + {Opt_gid, "gid=%u"}, + {Opt_umask, "umask=%o"}, + {Opt_prealloc_part, "prealloc_part=%u"}, + {Opt_prealloc_min, "prealloc_min=%u"}, + {Opt_prealloc_max, "prealloc_max=%u"}, + {Opt_xfer_size, "xfer_size=%u"}, + {Opt_buffer_size, "buffer_size=%u"}, + {Opt_prefetch, "prefetch=%u"}, + {Opt_sync_time, "sync_time=%u"}, + {Opt_no_checksums, "no_checksums"}, + {Opt_checksums, "checksums"}, + {Opt_ino64, "ino64=%s"}, + {Opt_usrquota, "usrquota"}, + {Opt_grpquota, "grpquota"}, + {Opt_err, NULL}, +}; + +__cold static int parse_opts(SPADFS *fs, char *opts, int remount) +{ + char *p; + while ((p = strsep(&opts, ","))) { + substring_t args[MAX_OPT_ARGS]; + int token, option; + char str[7]; + + if (!*p) + continue; + + token = match_token(p, tokens, args); + switch (token) { + case Opt_help: + return 2; + case Opt_uid: + if (match_int(args, &option)) + return 0; + if (remount) { + if (fs->uid != option) + return 0; + break; + } + fs->uid = option; + break; + case Opt_gid: + if (match_int(args, &option)) + return 0; + if (remount) { + if (fs->gid != option) + return 0; + break; + } + fs->gid = option; + break; + case Opt_umask: + if (match_octal(args, &option)) + return 0; + if (option < 0 || option > 0777) + return 0; + if (remount) { + if (fs->mode != (0777 & ~option)) + return 0; + break; + } + fs->mode = 0777 & ~option; + break; + case Opt_prealloc_part: + if (match_int(args, &option)) + return 0; + if (option <= 0) return 0; + set_prealloc_part(fs, option); + break; + case Opt_prealloc_min: + if (match_int(args, &option)) + return 0; + if (option < 0) return 0; + fs->min_prealloc = option; + break; + case Opt_prealloc_max: + if (match_int(args, &option)) + return 0; + if (option < 0) return 0; + fs->max_prealloc = option; + break; + case Opt_xfer_size: + if (match_int(args, &option)) + return 0; + if (option <= 0) return 0; + fs->xfer_size = option; + break; + case Opt_buffer_size: + if (match_int(args, &option)) + return 0; + if (option < 512 || option > PAGE_SIZE || (option & (option - 1))) + return 0; + if (remount) { + if (option != 512U << fs->sectors_per_buffer_bits) + return 0; + break; + } + fs->buffer_size = option; + break; + case Opt_prefetch: + if (match_int(args, &option)) + return 0; + if (option < 0) return 0; + fs->metadata_prefetch = option >> 9; + if (fs->metadata_prefetch & (fs->metadata_prefetch - 1)) + fs->metadata_prefetch = 1U << (fls(fs->metadata_prefetch) - 1); + break; + case Opt_sync_time: + if (match_int(args, &option)) + return 0; + if (option <= 0 || + option >= (1UL << (BITS_PER_LONG - 1)) / HZ) + return 0; + fs->spadfs_sync_time = option * HZ; + break; + case Opt_checksums: + fs->mount_flags |= + MOUNT_FLAGS_CHECKSUMS_OVERRIDE; + fs->mount_flags |= + MOUNT_FLAGS_CHECKSUMS; + break; + case Opt_no_checksums: + fs->mount_flags |= + MOUNT_FLAGS_CHECKSUMS_OVERRIDE; + fs->mount_flags &= + ~MOUNT_FLAGS_CHECKSUMS; + break; + case Opt_ino64: + match_strlcpy(str, args, sizeof str); + if (!strcmp(str, "no")) { + fs->mount_flags &= + ~(MOUNT_FLAGS_64BIT_INO | + MOUNT_FLAGS_64BIT_INO_FORCE); + } else if (!strcmp(str, "yes")) { + fs->mount_flags |= + MOUNT_FLAGS_64BIT_INO; + fs->mount_flags &= + ~MOUNT_FLAGS_64BIT_INO_FORCE; + } else if (!strcmp(str, "force")) { + fs->mount_flags |= + MOUNT_FLAGS_64BIT_INO | + MOUNT_FLAGS_64BIT_INO_FORCE; + } else { + return 0; + } + break; +#ifdef SPADFS_QUOTA + case Opt_usrquota: + fs->mount_flags |= MOUNT_FLAGS_USRQUOTA; + break; + case Opt_grpquota: + fs->mount_flags |= MOUNT_FLAGS_GRPQUOTA; + break; +#endif + default: + return 0; + } + } + return 1; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +static int spadfs_show_options(struct seq_file *seq, struct vfsmount *vfs) +{ + struct super_block *s = vfs->mnt_sb; +#else +static int spadfs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct super_block *s = root->d_sb; +#endif + SPADFS *fs = spadfs(s); + + if (fs->uid) + seq_printf(seq, ",uid=%u", (unsigned)fs->uid); + if (fs->gid) + seq_printf(seq, ",gid=%u", (unsigned)fs->gid); + seq_printf(seq, ",umask=%03o", (~fs->mode & 0777)); + seq_printf(seq, ",prealloc_part=%u", fs->prealloc_part); + seq_printf(seq, ",prealloc_min=%u", fs->min_prealloc); + seq_printf(seq, ",prealloc_max=%u", fs->max_prealloc); + seq_printf(seq, ",xfer_size=%u", fs->xfer_size); + seq_printf(seq, ",buffer_size=%u", 512U << fs->sectors_per_buffer_bits); + seq_printf(seq, ",prefetch=%u", fs->metadata_prefetch << 9); + seq_printf(seq, ",sync_time=%lu", fs->spadfs_sync_time / HZ); + if (fs->mount_flags & MOUNT_FLAGS_CHECKSUMS) + seq_printf(seq, ",checksums"); + else + seq_printf(seq, ",no_checksums"); + if (!(fs->mount_flags & MOUNT_FLAGS_64BIT_INO)) + seq_printf(seq, ",ino64=no"); + else if (!(fs->mount_flags & MOUNT_FLAGS_64BIT_INO_FORCE)) + seq_printf(seq, ",ino64=yes"); + else + seq_printf(seq, ",ino64=force"); +#ifdef SPADFS_QUOTA + if (fs->mount_flags & MOUNT_FLAGS_USRQUOTA) + seq_printf(seq, ",usrquota"); + if (fs->mount_flags & MOUNT_FLAGS_GRPQUOTA) + seq_printf(seq, ",grpquota"); +#endif + + return 0; +} + +static void spadfs_help(void) +{ + printk("\n\ +SPADFS filesystem options:\n\ + help display this text\n\ + uid=xxx set default uid of files\n\ + gid=xxx set default gid of files\n\ + umask=xxx set default mode of files\n\ + prealloc_part=xxx prealloc this part of existing file size\n\ + (i.e. 8 means prealloc 1/8 of file size)\n\ + prealloc_min=xxx minimum preallocation in bytes\n\ + prealloc_max=xxx maximum preallocation in bytes\n\ + xfer_size=xxx optimal request size reported in st_blksize\n\ + buffer_size=xxx set kernel buffer size\n\ + prefetch=xxx metadata prefetch in bytes\n\ + sync_time=xxx sync after this interval in seconds\n\ + no_checksums do not check and make checksums\n\ + checksums do check and make checksums\n\ + ino64=no,yes,force use 64-bit inode numbers\n\ + usrquota user quota\n\ + grpquota group quota\n\ +\n"); +} + +/* Report error. flags is the mask of TXFLAGS_* that will be set in txblock */ + +__cold void spadfs_error_(SPADFS *fs, unsigned flags, const char *msg, ...) +{ + unsigned long irqstate; + if (msg) { + va_list va; + va_start(va, msg); + vprintk(msg, va); + va_end(va); + dump_stack(); + } + spin_lock_irqsave(&fs->txflags_new_lock, irqstate); + fs->txflags_new |= flags; + spin_unlock_irqrestore(&fs->txflags_new_lock, irqstate); +} + +__cold static void *spadfs_alloc(size_t n_elements, size_t element_size, size_t extra) +{ + size_t size; + if (element_size && n_elements > ((size_t)-1 - extra) / element_size) + return NULL; + size = n_elements * element_size + extra; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0) && !TEST_RHEL_VERSION(7,5) + if (size <= PAGE_SIZE) + return kmalloc(size, GFP_KERNEL); + if (size <= KMALLOC_MAX_SIZE) { + void *ptr = kmalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); + if (ptr) + return ptr; + } + return vmalloc(size); +#else + return kvmalloc(size, GFP_KERNEL); +#endif +} + +#ifndef SIMPLE_SYNC_LOCK + +__cold static void spadfs_do_done_sync_lock(SPADFS *fs); + +__cold static int spadfs_do_init_sync_lock(SPADFS *fs) +{ + unsigned i; + for (i = 0; i < spadfs_nr_cpus; i++) { + const unsigned size = max((unsigned)sizeof(struct rw_semaphore), + (unsigned)L1_CACHE_BYTES); + int node = cpu_possible(i) ? cpu_to_node(i) : numa_node_id(); + /* (unsigned) kills warning */ + fs->sync_locks[i] = kmalloc_node(size, GFP_KERNEL, node); + if (!fs->sync_locks[i]) { + spadfs_do_done_sync_lock(fs); + return 1; + } + init_rwsem(fs->sync_locks[i]); + } + return 0; +} + +__cold static void spadfs_do_done_sync_lock(SPADFS *fs) +{ + unsigned i; + for (i = 0; i < spadfs_nr_cpus; i++) { + kfree(fs->sync_locks[i]); + fs->sync_locks[i] = NULL; + } +} + +void spadfs_do_down_read_sync_lock(SPADFS *fs, unsigned *cpu) +{ + /* + * Use raw_smp_processor_id() instead of smp_processor_id() to + * suppress warning message about using it with preempt enabled. + * Preempt doesn't really matter here, locking the lock on + * a different CPU will cause a small performance impact but + * no race condition. + */ + unsigned cp = raw_smp_processor_id(); + *cpu = cp; + down_read(fs->sync_locks[cp]); +} + +void spadfs_do_up_read_sync_lock(SPADFS *fs, unsigned cpu) +{ + up_read(fs->sync_locks[cpu]); +} + +void spadfs_do_down_write_sync_lock(SPADFS *fs) +{ + unsigned i; + for (i = 0; i < spadfs_nr_cpus; i++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + down_write_nested(fs->sync_locks[i], i); +#else + down_write(fs->sync_locks[i]); +#endif + } +} + +void spadfs_do_up_write_sync_lock(SPADFS *fs) +{ + unsigned i; + for (i = 0; i < spadfs_nr_cpus; i++) + up_write(fs->sync_locks[i]); +} + +#endif + +int spadfs_stop_cycles(SPADFS *fs, sector_t key, sector_t (*c)[2], + const char *msg) +{ + if (unlikely((*c)[0] == key && unlikely((*c)[1] != 0))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "cycle detected on key %Lx in %s", + (unsigned long long)key, msg); + return 1; + } + (*c)[1]++; + if (likely(!(((*c)[1] - 1) & (*c)[1]))) + (*c)[0] = key; + return 0; +} + +int spadfs_issue_flush(SPADFS *fs) +{ + int r; +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) + r = 0; +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35) + r = blkdev_issue_flush(fs->s->s_bdev, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37) + r = blkdev_issue_flush(fs->s->s_bdev, GFP_NOFS, NULL, BLKDEV_IFL_WAIT); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0) && !TEST_RHEL_VERSION(8,4) + r = blkdev_issue_flush(fs->s->s_bdev, GFP_NOFS, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) && !TEST_RHEL_VERSION(8,6) + r = blkdev_issue_flush(fs->s->s_bdev, GFP_NOFS); +#else + r = blkdev_issue_flush(fs->s->s_bdev); +#endif + if (unlikely(r == -EOPNOTSUPP)) + r = 0; + if (unlikely(r)) + spadfs_error(fs, TXFLAGS_IO_WRITE_ERROR, "flush error: %d", r); + return r; +} + +void spadfs_tx_block_checksum(struct txblock *tb) +{ + tb->checksum ^= CHECKSUM_BASE ^ __byte_sum(tb, 512); +} + +static int spadfs_increase_cc(SPADFS *fs, int inc) +{ + struct txblock *tb; + struct buffer_head *bh; + int r, rr = 0; + + tb = spadfs_read_tx_block(fs, &bh, "spadfs_increase_cc"); + if (unlikely(IS_ERR(tb))) + return PTR_ERR(tb); + + start_atomic_buffer_modify(fs, bh); + if (likely(inc == 1)) { + u16 cc = SPAD2CPU16_LV(&tb->cc); + if (cc == 0xffff) { + /* !!! FIXME: process wrap-around */ + spadfs_error(fs, TXFLAGS_FS_ERROR, "crash count wrap-around, remounting read-only"); + rr = 1; + goto skip_inc; + } + cc += inc; + CPU2SPAD16_LV(&tb->cc, cc); + if (unlikely(SPAD2CPU32_LV(&fs->cct[cc]))) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "crash count table is damaged (non-zero at %04x: %08x)", + (unsigned)SPAD2CPU16_LV(&tb->cc), + (unsigned)SPAD2CPU32_LV(&fs->cct[cc])); + rr = -EFSERROR; + } + fs->cc = cc - 1; + fs->txc = SPAD2CPU32_LV(&fs->cct[fs->cc]); + } else if (unlikely(inc == -1)) { + CPU2SPAD16_LV(&tb->cc, SPAD2CPU16_LV(&tb->cc) + inc); + /* If unmounting, clean up preallocated inodes */ + spin_lock(&fs->stable_ino_lock); + if (fs->stable_ino) { + CPU2SPAD64_LV(&tb->ino, fs->stable_ino); + fs->stable_ino = 0; + } + spin_unlock(&fs->stable_ino_lock); + } + +skip_inc: + spin_lock_irq(&fs->txflags_new_lock); + fs->txflags |= fs->txflags_new; + spin_unlock_irq(&fs->txflags_new_lock); + + CPU2SPAD32_LV(&tb->txflags, fs->txflags); + spadfs_tx_block_checksum(tb); + end_atomic_buffer_modify(fs, bh); + + r = spadfs_sync_dirty_buffer(bh); + if (likely(!r)) + r = spadfs_issue_flush(fs); + if (unlikely(r)) + spadfs_error(fs, TXFLAGS_IO_WRITE_ERROR, + "write error at tx block: %d", r); + + spadfs_brelse(fs, bh); + return unlikely(r) ? r : rr; +} + +static int spadfs_update_error_flags(SPADFS *fs) +{ + /* + * Don't lock when reading txflags_new --- if we miss an update here, + * it would be updated later. + */ + if (unlikely(fs->txflags_new & ~fs->txflags)) { + if (sb_rdonly(fs->s)) + return 0; + return spadfs_increase_cc(fs, 0); + } + return 0; +} + +static noinline int spadfs_new_stable_ino_slow(SPADFS *fs, u64 *stable_ino) +{ + int r; + struct txblock *tb; + struct buffer_head *bh; + u64 next_stable_ino; + + /*sync_lock_decl*/ + down_write_sync_lock(fs); + + spin_lock(&fs->stable_ino_lock); + *stable_ino = fs->stable_ino; + if (unlikely((*stable_ino & (SPADFS_INO_BATCH - 1)) != 0)) { + fs->stable_ino++; + spin_unlock(&fs->stable_ino_lock); + r = 0; + goto unlock_ret_r; + } + spin_unlock(&fs->stable_ino_lock); + + tb = spadfs_read_tx_block(fs, &bh, "spadfs_new_stable_ino_slow"); + if (unlikely(IS_ERR(tb))) { + r = PTR_ERR(tb); + goto unlock_ret_r; + } + + start_atomic_buffer_modify(fs, bh); + + *stable_ino = SPAD2CPU64_LV(&tb->ino); + if (unlikely(!*stable_ino)) + *stable_ino = SPADFS_INO_ROOT + 1; + next_stable_ino = (*stable_ino | (SPADFS_INO_BATCH - 1)) + 1; + if (unlikely(next_stable_ino == SPADFS_INO_INITIAL_REGION)) + next_stable_ino = (u64)SPADFS_INO_INITIAL_REGION * 2; + if (unlikely(!(next_stable_ino & 0xFFFFFFFF))) + next_stable_ino |= (u64)SPADFS_INO_INITIAL_REGION * 2; + CPU2SPAD64_LV(&tb->ino, next_stable_ino); + + spin_lock(&fs->stable_ino_lock); + fs->stable_ino = *stable_ino + 1; + spin_unlock(&fs->stable_ino_lock); + + spadfs_tx_block_checksum(tb); + + end_atomic_buffer_modify(fs, bh); + spadfs_brelse(fs, bh); + + r = 0; + +unlock_ret_r: + up_write_sync_lock(fs); + return r; +} + +int spadfs_new_stable_ino(SPADFS *fs, u64 *stable_ino) +{ + if (unlikely(sb_rdonly(fs->s))) + return -EROFS; + spin_lock(&fs->stable_ino_lock); + if (unlikely(!((*stable_ino = fs->stable_ino) & + (SPADFS_INO_BATCH - 1)))) { + spin_unlock(&fs->stable_ino_lock); + return spadfs_new_stable_ino_slow(fs, stable_ino); + } + fs->stable_ino++; + spin_unlock(&fs->stable_ino_lock); + return 0; +} + +static int spadfs_start_new_tx(SPADFS *fs) +{ + int r; + if (unlikely(!fs->max_allocation)) + fs->max_allocation = 1U << fs->sectors_per_disk_block_bits; + if (unlikely(SPAD2CPU32_LV(&fs->cct[fs->cc]) == 0x7fffffff)) { + if (unlikely(r = spadfs_increase_cc(fs, 1))) { + if (r > 0) + return 0; /* it is read-only now */ + return r; + } + } else { + if (unlikely(r = spadfs_update_error_flags(fs))) + return r; + } + CPU2SPAD32_LV(&fs->cct[fs->cc], SPAD2CPU32_LV(&fs->cct[fs->cc]) + 1); + fs->txc = SPAD2CPU32_LV(&fs->cct[fs->cc]); + return 0; +} + +int spadfs_commit_unlocked(SPADFS *fs) +{ + int r, rr; + unsigned sector; + void *cct_data; + struct buffer_head *bh; + + assert_write_sync_lock(fs); + + fs->commit_sequence++; + + if (unlikely(sb_rdonly(fs->s))) + return -EFSERROR; + + while (unlikely(!list_empty(&fs->clear_list))) { + SPADFNODE *f = list_entry(fs->clear_list.prev, SPADFNODE, clear_entry); + spadfs_clear_last_block(&f->vfs_inode); + } + + mutex_lock(&fs->alloc_lock); +#ifdef SPADFS_META_PREALLOC + spadfs_prealloc_discard_unlocked(fs, &fs->meta_prealloc); +#endif + spadfs_prealloc_discard_unlocked(fs, &fs->small_prealloc); + if (unlikely(fs->max_freed_run > fs->max_allocation)) + fs->max_allocation = fs->max_freed_run; + fs->max_freed_run = 0; + mutex_unlock(&fs->alloc_lock); + +#ifdef SPADFS_QUOTA + mutex_lock(&fs->quota_alloc_lock); +#endif + + r = sync_blockdev(fs->s->s_bdev); + if (unlikely(r)) + spadfs_error(fs, TXFLAGS_IO_WRITE_ERROR, "write error: %d", r); + + sector = fs->cc / (512 / 4); + sector &= ~((1U << fs->sectors_per_buffer_bits) - 1); + cct_data = spadfs_get_new_sector(fs, fs->cct_sec + sector, &bh, + "spadfs_commit_unlocked"); + if (unlikely(IS_ERR(cct_data))) { + if (!r) + r = PTR_ERR(cct_data); + goto unlock_ret; + } + + start_atomic_buffer_modify(fs, bh); + memcpy(cct_data, (char *)fs->cct + (sector << 9), 512U << fs->sectors_per_buffer_bits); + end_atomic_buffer_modify(fs, bh); + + rr = spadfs_sync_dirty_buffer(bh); + spadfs_brelse(fs, bh); + + if (likely(!rr)) + rr = spadfs_issue_flush(fs); + + if (unlikely(rr != 0)) { + spadfs_error(fs, TXFLAGS_IO_WRITE_ERROR, + "write error in crash count table: %d", rr); + spadfs_update_error_flags(fs); + if (!r) + r = rr; + goto unlock_ret; + } + + rr = spadfs_start_new_tx(fs); + if (unlikely(rr)) { + if (!r) + r = rr; + } + + if (likely(!r)) + fs->need_background_sync = 0; + +unlock_ret: + +#ifdef SPADFS_QUOTA + mutex_unlock(&fs->quota_alloc_lock); +#endif + + return r; +} + +int spadfs_commit(SPADFS *fs) +{ + /*sync_lock_decl*/ + int r, rr; + + if (unlikely(sb_rdonly(fs->s))) + return 0; + + /* + * Improves performance very much --- do most of the syncing + * outside the lock. + */ + r = sync_blockdev(fs->s->s_bdev); + if (unlikely(r)) + spadfs_error(fs, TXFLAGS_IO_WRITE_ERROR, "write error: %d", r); + + down_write_sync_lock(fs); + rr = spadfs_commit_unlocked(fs); + if (unlikely(rr)) + r = rr; + up_write_sync_lock(fs); + + return r; +} + +static int spadfs_sync_fs(struct super_block *s, int wait) +{ + SPADFS *fs = spadfs(s); + if (!READ_ONCE(fs->need_background_sync)) + return 0; + else if (!wait) + return filemap_fdatawrite(s->s_bdev->bd_inode->i_mapping); + else + return spadfs_commit(spadfs(s)); +} + +#ifndef NEW_WORKQUEUE +static void spadfs_sync_work(void *fs_) +{ + SPADFS *fs = fs_; +#else +static void spadfs_sync_work(struct work_struct *w) +{ + SPADFS *fs = container_of(w, SPADFS, spadfs_sync_work.work); +#endif + spadfs_reclaim_max_allocation(fs); + if (READ_ONCE(fs->need_background_sync)) { + spadfs_commit(fs); + } + queue_delayed_work(fs->spadfs_syncer, &fs->spadfs_sync_work, + fs->spadfs_sync_time); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) +static int spadfs_statfs(struct dentry *dentry, struct kstatfs *ks) +{ + struct super_block *s = dentry->d_sb; +#else +static int spadfs_statfs(struct super_block *s, struct kstatfs *ks) +{ +#endif + SPADFS *fs = spadfs(s); + sector_t freespace; + + mutex_lock(&fs->alloc_lock); + + ks->f_type = s->s_magic; + ks->f_bsize = 512U << fs->sectors_per_disk_block_bits; + ks->f_blocks = fs->size >> fs->sectors_per_disk_block_bits; + + freespace = spadfs_get_freespace(fs); + if (likely(freespace >= fs->alloc_mem_sectors)) + freespace -= fs->alloc_mem_sectors; + else + freespace = 0; + + ks->f_bfree = freespace >> fs->sectors_per_disk_block_bits; + ks->f_bavail = likely(freespace >= fs->reserve_sectors) ? + (freespace - fs->reserve_sectors) >> fs->sectors_per_disk_block_bits : + 0; + + ks->f_files = (u64)fs->zones[0].grp_n << (fs->sectors_per_group_bits - fs->sectors_per_disk_block_bits); + if (unlikely(ks->f_files > ks->f_blocks)) + ks->f_files = ks->f_blocks; + ks->f_ffree = fs->zones[0].freespace >> fs->sectors_per_disk_block_bits; + + ks->f_namelen = MAX_NAME_LEN; + + mutex_unlock(&fs->alloc_lock); + + return 0; +} + +__cold static int spadfs_remount_fs(struct super_block *s, int *flags, char *data) +{ + int r, o; + SPADFS *fs = spadfs(s); + /*sync_lock_decl*/ + + down_write_sync_lock(fs); + + o = parse_opts(fs, data, 1); + if (!o) { + r = -EINVAL; + goto unlock_ret_r; + } + if (o == 2) { + spadfs_help(); + r = -EAGAIN; + goto unlock_ret_r; + } + + if ((*flags ^ s->s_flags) & SB_RDONLY) { + if (*flags & SB_RDONLY) { +#ifdef SPADFS_QUOTA + if ((r = dquot_suspend(s, -1))) { + *flags &= ~SB_RDONLY; + goto unlock_ret_r; + } +#endif + if ((r = spadfs_commit_unlocked(fs))) + goto unlock_ret_r; + if ((r = spadfs_increase_cc(fs, -1))) + goto unlock_ret_r; + CPU2SPAD32_LV(&fs->cct[fs->cc], + SPAD2CPU32_LV(&fs->cct[fs->cc]) - 1); + } else { + if ((r = spadfs_increase_cc(fs, 1))) { + if (r < 0) + goto unlock_ret_r; + *flags |= SB_RDONLY; + } + if ((r = spadfs_start_new_tx(fs))) + goto unlock_ret_r; +#ifdef SPADFS_QUOTA + dquot_resume(s, -1); +#endif + } + } + + r = 0; + +unlock_ret_r: + up_write_sync_lock(fs); + return r; +} + +#if defined(SPADFS_QUOTA) && SPADFS_QUOTA >= 3 +static const struct quotactl_ops spadfs_quotactl_ops = { + .quota_on = dquot_quota_on, + .quota_off = dquot_quota_off, + .quota_sync = dquot_quota_sync, + .get_state = dquot_get_state, + .set_info = dquot_set_dqinfo, + .get_dqblk = dquot_get_dqblk, + .set_dqblk = dquot_set_dqblk, + .get_nextdqblk = dquot_get_next_dqblk, +}; +#endif + +__cold static int spadfs_fill_super(struct super_block *s, void *options, int silent) +{ + SPADFS *fs; + const char *wq_name = +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,13) + "spadfssync"; +#else + "spadfs_syncer"; +#endif + int blocksize; + int r; + int o; + unsigned i, n; + sector_t sec; + struct buffer_head *bh; + struct superblock *sb; + struct txblock *tb; + const char *msg; + int cycle; + struct inode *root; + size_t sizeof_spadfs = sizeof(SPADFS) +#ifndef SIMPLE_SYNC_LOCK + + spadfs_nr_cpus * sizeof(struct rw_semaphore *) +#endif + ; + + sb = kmalloc(512, GFP_KERNEL); + if (unlikely(!sb)) { + r = -ENOMEM; + goto err0; + } + + if (unlikely(!(fs = spadfs_alloc(1, sizeof_spadfs, 0)))) { + r = -ENOMEM; + goto err1; + } + + s->s_fs_info = fs; + s->s_flags |= SB_NOATIME; + s->s_magic = MEMORY_SUPER_MAGIC; + s->s_op = &spadfs_sops; +#ifdef SPADFS_QUOTA + s->dq_op = &dquot_operations; +#if SPADFS_QUOTA >= 3 + s->s_qcop = &spadfs_quotactl_ops; +#endif +#if SPADFS_QUOTA >= 2 + s->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP; +#endif +#endif +#ifdef SPADFS_XATTR + s->s_xattr = +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35) + (struct xattr_handler **) +#endif + spadfs_xattr_handlers; +#endif + if (sizeof(sector_t) == 4) + s->s_maxbytes = ((u64)512 << 32) - 512; + else + s->s_maxbytes = ((u64)512 << 48) - 512; + if (s->s_maxbytes > MAX_LFS_FILESIZE) + s->s_maxbytes = MAX_LFS_FILESIZE; + memset(fs, 0, sizeof_spadfs); + fs->s = s; + mutex_init(&fs->alloc_lock); + spadfs_allocmem_init(fs); + fs->alloc_reservations = RB_ROOT; + mutex_init(&fs->inode_list_lock); + INIT_LIST_HEAD(&fs->apage_lru); +#ifdef SPADFS_QUOTA + mutex_init(&fs->quota_alloc_lock); +#endif + spin_lock_init(&fs->stable_ino_lock); + spin_lock_init(&fs->txflags_new_lock); + fs->commit_sequence = 1; + mutex_init(&fs->trim_lock); + + spin_lock_init(&fs->clear_lock); + INIT_LIST_HEAD(&fs->clear_list); + + spadfs_buffer_leaks_init(fs); + + init_sync_lock(fs, r = -ENOMEM; goto err2;); + + fs->uid = get_current_uid(); + fs->gid = get_current_gid(); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,30) + fs->mode = 0777 & ~current->fs->umask; +#else + fs->mode = 0777 & ~current_umask(); +#endif + set_prealloc_part(fs, SPADFS_DEFAULT_PREALLOC_PART); + fs->min_prealloc = SPADFS_DEFAULT_MIN_PREALLOC; + fs->max_prealloc = SPADFS_DEFAULT_MAX_PREALLOC; + fs->xfer_size = 0; + fs->metadata_prefetch = SPADFS_DEFAULT_METADATA_PREFETCH >> 9; + fs->spadfs_sync_time = SPADFS_DEFAULT_SYNC_TIME; + +#ifdef CHECK_ALLOCMEM + /* + * Unit test for allocmem. It is normally not used much, so we + * must stress-test it here. + */ + r = spadfs_allocmem_unit_test(fs); + if (r) + goto err2; +#endif + + fs->inode_list = spadfs_alloc(SPADFS_INODE_HASH_SIZE, sizeof(struct hlist_node), 0); + if (!fs->inode_list) { + r = -ENOMEM; + goto err2; + } + for (i = 0; i < SPADFS_INODE_HASH_SIZE; i++) + INIT_HLIST_HEAD(&fs->inode_list[i]); + + o = parse_opts(fs, options, 0); + if (!o) { + r = -EINVAL; + goto err2; + } + if (o == 2) { + spadfs_help(); + r = -EAGAIN; + goto err2; + } + + if (!fs->buffer_size) { + blocksize = sb_min_blocksize(s, 512); + if (unlikely(!blocksize)) { + if (!silent) + printk(KERN_ERR "spadfs: unable to set blocksize\n"); + r = -EIO; + goto err2; + } + } else { + if (unlikely(!sb_set_blocksize(s, fs->buffer_size))) { + printk(KERN_ERR "spadfs: can't set blocksize %d\n", fs->buffer_size); + r = -EINVAL; + goto err2; + } + blocksize = fs->buffer_size; + } + + cycle = 0; + +read_again: + if (unlikely(blocksize & (blocksize - 1)) || unlikely(blocksize < 512)) + panic("spadfs: wrong blocksize: %d", blocksize); + fs->sectors_per_buffer_bits = ffs(blocksize) - 1 - 9; + fs->size = (sector_t)-1; + + bh = sb_bread(s, SUPERBLOCK_SECTOR >> fs->sectors_per_buffer_bits); + if (!bh) { + if (!silent) + printk(KERN_ERR "spadfs: unable to read super block\n"); + r = -EIO; + goto err2; + } + memcpy(sb, spadfs_buffer_data(fs, SUPERBLOCK_SECTOR, bh), 512); + __brelse(bh); + + if (unlikely(memcmp(sb->signature, SUPERBLOCK_SIGNATURE, sizeof sb->signature))) { + if (!silent) + printk(KERN_ERR "spadfs: superblock not found\n"); + r = -EINVAL; + goto err2; + } + if (SPAD2CPU64_LV(&sb->byte_sex) != 0x0123456789ABCDEFLL) { + if (!silent) + printk(KERN_ERR "spadfs: byte sex does not match: %16llx\n", + (unsigned long long)SPAD2CPU64_LV(&sb->byte_sex)); + r = -EINVAL; + goto err2; + } + if (unlikely(sb->version_major != SPADFS_VERSION_MAJOR)) { + if (!silent) + printk(KERN_ERR "spadfs: bad major version number (disk %d.%d.%d, driver %d.%d.%d)\n", + sb->version_major, + sb->version_middle, + sb->version_minor, + SPADFS_VERSION_MAJOR, + SPADFS_VERSION_MIDDLE, + SPADFS_VERSION_MINOR); + r = -EINVAL; + goto err2; + } + if (unlikely(SPAD2CPU32_LV(&sb->flags_compat_rw) & ~(0)) || + unlikely(SPAD2CPU32_LV(&sb->flags_compat_ro) & ~( + FLAG_COMPAT_RO_DIRECTORY_SIZES)) || + unlikely(SPAD2CPU32_LV(&sb->flags_compat_none) & ~( + FLAG_COMPAT_NONE_UNIX_NAMES | + FLAG_COMPAT_NONE_DYNAMIC_MAGIC | + FLAG_COMPAT_NONE_EXTRA_SPACE))) { + if (!silent) + printk(KERN_ERR "spadfs: incompatible feature flags in superblock: %08x/%08x/%08x/%08x\n", + (unsigned)SPAD2CPU32_LV(&sb->flags_compat_fsck), + (unsigned)SPAD2CPU32_LV(&sb->flags_compat_rw), + (unsigned)SPAD2CPU32_LV(&sb->flags_compat_ro), + (unsigned)SPAD2CPU32_LV(&sb->flags_compat_none)); + r = -EINVAL; + goto err2; + } + if (unlikely(__byte_sum(sb, 512) != CHECKSUM_BASE)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, "bad checksum on superblock: %02x", __byte_sum(sb, 512) ^ CHECKSUM_BASE); + r = -EFSERROR; + goto err2; + } + if (unlikely((u64)(sector_t)SPAD2CPU64_LV(&sb->size) != SPAD2CPU64_LV(&sb->size))) { + if (!silent) + printk(KERN_ERR "spadfs: 48-bit filesystem not supported by vfs layer - enable it in kernel configuration\n"); + r = -EINVAL; + goto err2; + } + if (unlikely((msg = validate_super(sb)) != NULL)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, "invalid superblock: %s", msg); + r = -EFSERROR; + goto err2; + } + fs->sectors_per_disk_block_bits = sb->sectors_per_block_bits; + if (!fs->buffer_size) { + unsigned char wanted_sectors_per_buffer_bits = min(fs->sectors_per_disk_block_bits, (unsigned char)(PAGE_SHIFT - 9)); + if (unlikely(fs->sectors_per_buffer_bits != wanted_sectors_per_buffer_bits)) { + if (unlikely(cycle)) { + if (!silent) + printk(KERN_ERR "spadfs: superblock changed while setting device size\n"); + r = -EIO; + goto err2; + } + blocksize = 512U << wanted_sectors_per_buffer_bits; + if (unlikely(!sb_set_blocksize(s, blocksize))) { + if (!silent) + printk(KERN_ERR "spadfs: can't set blocksize %d\n", blocksize); + r = -EINVAL; + goto err2; + } + cycle = 1; + goto read_again; + } + } else { + if (fs->sectors_per_disk_block_bits < fs->sectors_per_buffer_bits) { + if (!silent) + printk(KERN_ERR "spadfs: buffer size (%u) is greater than block size (%u)\n", + 512U << fs->sectors_per_buffer_bits, 512U << fs->sectors_per_disk_block_bits); + r = -EINVAL; + goto err2; + } + } + + fs->sectors_per_fnodepage_bits = sb->sectors_per_fnodepage_bits; + fs->sectors_per_page_bits = sb->sectors_per_page_bits; + fs->pagesize_bits = fs->sectors_per_page_bits + 9; + fs->apage_size = 1U << (fs->pagesize_bits - 1); + fs->dnode_hash_bits = fs->pagesize_bits - DNODE_PAGE_ENTRY_BITS - 1; + fs->dnode_data_size = DNODE_PAGE_ENTRY_SIZE << fs->dnode_hash_bits; + fs->dnode_page_sectors = spadfs_roundup_blocksize(fs, + DNODE_ENTRY_OFFSET + (fs->dnode_data_size << 1)); + fs->dnode_page_sectors >>= 9; + fs->sectors_per_cluster_bits = sb->sectors_per_cluster_bits; + fs->cluster_threshold = (512U << fs->sectors_per_cluster_bits) * + SPAD2CPU16_LV(&sb->cluster_threshold); + fs->sectors_per_group_bits = sb->sectors_per_group_bits; + if (fs->sectors_per_group_bits > sizeof(sector_t) * 8 - 1) + fs->sectors_per_group_bits = sizeof(sector_t) * 8 - 1; + fs->group_mask = ((sector_t)1 << fs->sectors_per_group_bits) - 1; + fs->n_apage_mappings = fs->apage_size >> (fs->sectors_per_buffer_bits + 9); + if (unlikely(!fs->n_apage_mappings)) + fs->n_apage_mappings = 1; + + fs->txb_sec = SPAD2CPU64_LV(&sb->txblock); + fs->apage_index0_sec = SPAD2CPU64_LV(&sb->apage_index[0]); + fs->apage_index1_sec = SPAD2CPU64_LV(&sb->apage_index[1]); + fs->cct_sec = SPAD2CPU64_LV(&sb->cct); + fs->root_sec = SPAD2CPU64_LV(&sb->root); + fs->size = SPAD2CPU64_LV(&sb->size); + fs->reserve_sectors = SPAD2CPU64_LV(&sb->reserve_sectors); + + fs->flags_compat_fsck = SPAD2CPU32_LV(&sb->flags_compat_fsck); + fs->flags_compat_rw = SPAD2CPU32_LV(&sb->flags_compat_rw); + fs->flags_compat_ro = SPAD2CPU32_LV(&sb->flags_compat_ro); + fs->flags_compat_none = SPAD2CPU32_LV(&sb->flags_compat_none); + + if (fs->mount_flags & MOUNT_FLAGS_CHECKSUMS_OVERRIDE) { + if (fs->mount_flags & MOUNT_FLAGS_CHECKSUMS) + fs->flags_compat_fsck &= ~FLAG_COMPAT_FSCK_NO_CHECKSUMS; + else + fs->flags_compat_fsck |= FLAG_COMPAT_FSCK_NO_CHECKSUMS; + } + + if (!fs->xfer_size) { + fs->xfer_size = fs->cluster_threshold; + if (fs->xfer_size > SPADFS_DEFAULT_MAX_XFER_SIZE) + fs->xfer_size = SPADFS_DEFAULT_MAX_XFER_SIZE; + } + for (i = 512; i < INT_MAX / 2; i <<= 1) + if (i >= fs->xfer_size) + break; + fs->xfer_size = i; + + if (unlikely(IS_ERR(tb = spadfs_read_tx_block(fs, &bh, + "spadfs_fill_super 1")))) { + r = PTR_ERR(tb); + goto err2; + } + fs->cc = SPAD2CPU16_LV(&tb->cc); + fs->a_cc = tb->a_cc; /* fs->a_cc and fs->a_txc are in disk-endianity */ + fs->a_txc = tb->a_txc; + fs->txflags = SPAD2CPU32_LV(&tb->txflags); + + if (unlikely(fs->txflags & TXFLAGS_DIRTY)) + spadfs_error(fs, 0, + "filesystem was not cleanly unmounted"); + if (unlikely(fs->txflags & TXFLAGS_IO_READ_ERROR)) + spadfs_error(fs, 0, + "filesystem has encountered read i/o error"); + if (unlikely(fs->txflags & TXFLAGS_IO_WRITE_ERROR)) + spadfs_error(fs, 0, + "filesystem has encountered write i/o error"); + if (unlikely(fs->txflags & TXFLAGS_FS_ERROR)) + spadfs_error(fs, 0, + "filesystem has errors in structures"); + if (unlikely(fs->txflags & TXFLAGS_CHECKSUM_ERROR)) + spadfs_error(fs, 0, + "filesystem structures have bad checksums"); + if (unlikely(fs->txflags & TXFLAGS_EA_ERROR)) + spadfs_error(fs, 0, + "filesystem structures has errors in extended attributes"); + if (unlikely(fs->txflags & ~( + TXFLAGS_DIRTY | + TXFLAGS_IO_READ_ERROR | + TXFLAGS_IO_WRITE_ERROR | + TXFLAGS_FS_ERROR | + TXFLAGS_CHECKSUM_ERROR | + TXFLAGS_EA_ERROR))) + spadfs_error(fs, 0, + "filesystem has unknown error flags %08x", + fs->txflags); + if (unlikely(fs->txflags) && !sb_rdonly(s)) + spadfs_error(fs, 0, "running spadfsck is recommended"); + + spadfs_brelse(fs, bh); + + if (unlikely(!(fs->cct = spadfs_alloc(1, CCT_SIZE, 0)))) { + r = -ENOMEM; + goto err3; + } + + for (i = 0; i < CCT_SIZE / 512; i += 1U << fs->sectors_per_buffer_bits) { + void *p = spadfs_read_sector(fs, fs->cct_sec + i, &bh, + CCT_SIZE / 512 - i, "spadfs_fill_super 2"); + if (unlikely(IS_ERR(p))) { + r = PTR_ERR(p); + goto err3; + } + memcpy((u8 *)fs->cct + (i << 9), p, 512U << fs->sectors_per_buffer_bits); + spadfs_brelse(fs, bh); + } + fs->txc = SPAD2CPU32_LV(&fs->cct[fs->cc]); + + fs->n_apages = N_APAGES(fs->size, fs->pagesize_bits, fs->sectors_per_disk_block_bits + 9); + n = APAGE_INDEX_SECTORS(fs->n_apages, 512U << fs->sectors_per_disk_block_bits); + + fs->apage_index = spadfs_alloc(n, 512, sizeof(struct apage_index_entry)); + if (unlikely(!fs->apage_index)) { + r = -ENOMEM; + goto err3; + } + memset(fs->apage_index, 0, sizeof(struct apage_index_entry)); + fs->apage_index++; + + sec = CC_VALID(fs, &fs->a_cc, &fs->a_txc) ? + fs->apage_index0_sec : fs->apage_index1_sec; + for (i = 0; i < n; + i += 1U << fs->sectors_per_buffer_bits, + sec += 1U << fs->sectors_per_buffer_bits) { + void *p = spadfs_read_sector(fs, sec, &bh, n - i, "spadfs_fill_super 3"); + if (unlikely(IS_ERR(p))) { + r = PTR_ERR(p); + goto err3; + } + memcpy((u8 *)fs->apage_index + ((unsigned long)i << 9), p, 512U << fs->sectors_per_buffer_bits); + spadfs_brelse(fs, bh); + } + + fs->apage_info = spadfs_alloc(fs->n_apages, sizeof(struct apage_info), 0); + if (unlikely(!fs->apage_info)) { + r = -ENOMEM; + goto err3; + } + for (i = 0; i < fs->n_apages; i++) { + fs->apage_info[i].mapping[0].map = NULL; + spadfs_cond_resched(); + } + for (i = 0; i < fs->n_apages; i++) { + APAGE_MAP *m; + if (unlikely(!(m = kmalloc(fs->n_apage_mappings * 2 * sizeof(APAGE_MAP), GFP_KERNEL)))) { + r = -ENOMEM; + goto err3; + } + fs->apage_info[i].mapping[0].map = m; + fs->apage_info[i].mapping[1].map = m + fs->n_apage_mappings; + memset(m, 0, fs->n_apage_mappings * 2 * sizeof(APAGE_MAP)); + spadfs_cond_resched(); + } + + fs->n_groups = (fs->size + (u64)fs->group_mask) >> fs->sectors_per_group_bits; + fs->zones[0].grp_start = 0; + fs->zones[0].grp_n = SPAD2CPU16_LV(&sb->small_file_group); + fs->zones[1].grp_start = SPAD2CPU16_LV(&sb->small_file_group); + fs->zones[1].grp_n = SPAD2CPU16_LV(&sb->large_file_group) - + SPAD2CPU16_LV(&sb->small_file_group); + fs->zones[2].grp_start = SPAD2CPU16_LV(&sb->large_file_group); + fs->zones[2].grp_n = fs->n_groups - + SPAD2CPU16_LV(&sb->large_file_group); + + fs->group_info = spadfs_alloc(fs->n_groups, sizeof(struct spadfs_group_info), 0); + if (unlikely(!fs->group_info)) { + r = -ENOMEM; + goto err3; + } + memset(fs->group_info, 0, fs->n_groups * sizeof(struct spadfs_group_info)); + for (i = 0; i < fs->n_groups; i++) { + if (i < fs->zones[1].grp_start) + fs->group_info[i].zone = &fs->zones[0]; + else if (i < fs->zones[2].grp_start) + fs->group_info[i].zone = &fs->zones[1]; + else + fs->group_info[i].zone = &fs->zones[2]; + } + + fs->tmp_map = kmalloc(fs->n_apage_mappings * sizeof(APAGE_MAP), + GFP_KERNEL); + if (unlikely(!fs->tmp_map)) { + r = -ENOMEM; + goto err3; + } + memset(fs->tmp_map, 0, fs->n_apage_mappings * sizeof(APAGE_MAP)); + for (i = 0; i < fs->n_apage_mappings; i++) { + fs->tmp_map[i].entry = kmalloc(512U << fs->sectors_per_buffer_bits, GFP_KERNEL); + if (unlikely(!fs->tmp_map[i].entry)) { + r = -ENOMEM; + goto err3; + } + } + + for (i = 0; i < fs->n_apages; i++) + if (!SPAD2CPU64_LV(&fs->apage_index[i].end_sector)) + break; + fs->n_active_apages = i; + + r = spadfs_count_free_space(fs); + if (r) + goto err3; + + if (!sb_rdonly(s)) { + if ((r = spadfs_increase_cc(fs, 1))) { + if (r < 0) + goto err3; + s->s_flags |= SB_RDONLY; + } + if ((r = spadfs_start_new_tx(fs))) + goto err4; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,38) + if (unlikely(!(fs->flags_compat_none & FLAG_COMPAT_NONE_UNIX_NAMES))) + s->s_d_op = &spadfs_dops; +#endif + root = spadfs_iget(s, make_fixed_spadfs_ino_t(fs->root_sec), 0, 0); + if (unlikely(IS_ERR(root))) { + r = PTR_ERR(root); + goto err4; + } +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + s->s_root = d_alloc_root(root); +#else + s->s_root = d_make_root(root); +#endif + if (unlikely(!s->s_root)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + iput(root); +#endif + printk(KERN_ERR "spadfs: unable to allocate root dentry\n"); + r = -ENOMEM; + goto err4; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) + spadfs_set_dentry_operations(fs, s->s_root); +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) + fs->spadfs_syncer = create_singlethread_workqueue(wq_name); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) + fs->spadfs_syncer = create_freezeable_workqueue(wq_name); +#else + fs->spadfs_syncer = create_freezable_workqueue(wq_name); +#endif + if (unlikely(!fs->spadfs_syncer)) { + r = -ENOMEM; + goto err4; + } +#ifndef NEW_WORKQUEUE + INIT_WORK(&fs->spadfs_sync_work, spadfs_sync_work, fs); +#else + INIT_DELAYED_WORK(&fs->spadfs_sync_work, spadfs_sync_work); +#endif + queue_delayed_work(fs->spadfs_syncer, &fs->spadfs_sync_work, fs->spadfs_sync_time); + + kfree(sb); + + return 0; + +err4: + if (!sb_rdonly(s)) + spadfs_increase_cc(fs, -1); +err3: + spadfs_update_error_flags(fs); +err2: + spadfs_free_super(s); +err1: + kfree(sb); +err0: + return r; +} + +__cold static void spadfs_free_super(struct super_block *s) +{ + SPADFS *fs = spadfs(s); + unsigned i; + + if (fs->spadfs_syncer) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) + cancel_rearming_delayed_workqueue(fs->spadfs_syncer, + &fs->spadfs_sync_work); +#else + cancel_delayed_work_sync(&fs->spadfs_sync_work); +#endif + destroy_workqueue(fs->spadfs_syncer); + } + + spadfs_allocmem_done(fs); + + if (fs->apage_info) { + spadfs_prune_cached_apage_buffers(fs); + for (i = 0; i < fs->n_apages; i++) { + kfree(fs->apage_info[i].mapping[0].map); + spadfs_cond_resched(); + } + kvfree(fs->apage_info); + } + spadfs_cond_resched(); + + if (fs->tmp_map) { + for (i = 0; i < fs->n_apage_mappings; i++) + kfree(fs->tmp_map[i].entry); + kfree(fs->tmp_map); + } + spadfs_cond_resched(); + + kvfree(fs->group_info); + spadfs_cond_resched(); + + if (fs->apage_index) + kvfree(fs->apage_index - 1); + spadfs_cond_resched(); + + kvfree(fs->cct); + + if (fs->inode_list) { + for (i = 0; i < SPADFS_INODE_HASH_SIZE; i++) + if (unlikely(!hlist_empty(&fs->inode_list[i]))) + panic("spadfs: spadfs_free_super: inodes leaked"); + kvfree(fs->inode_list); + } + + done_sync_lock(fs); + + spadfs_buffer_leaks_done(fs); + + mutex_destroy(&fs->alloc_lock); + mutex_destroy(&fs->inode_list_lock); +#ifdef SPADFS_QUOTA + mutex_destroy(&fs->quota_alloc_lock); +#endif + mutex_destroy(&fs->trim_lock); + + kvfree(fs); + + s->s_fs_info = NULL; +} + +__cold static void spadfs_put_super(struct super_block *s) +{ + SPADFS *fs = spadfs(s); + +#ifdef SPADFS_QUOTA + dquot_disable(s, -1, DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED); +#endif + + /* + * If it didn't commit due to I/O error, do not decrease crash count + * --- act as if crash happened + */ + if (!spadfs_commit(fs)) { + if (fs->mount_flags & MOUNT_FLAGS_CLEAR_DIRTY_ON_UNMOUNT) { + fs->txflags &= ~TXFLAGS_DIRTY; + spin_lock_irq(&fs->txflags_new_lock); + fs->txflags_new &= ~TXFLAGS_DIRTY; + spin_unlock_irq(&fs->txflags_new_lock); + } + if (!sb_rdonly(s)) + spadfs_increase_cc(fs, -1); + } else { + spadfs_error(fs, TXFLAGS_IO_WRITE_ERROR, + "i/o error at unmount - not commiting"); + spadfs_update_error_flags(fs); + } + /*{ + int i; + printk("groups: "); + for (i = 0; i < fs->n_groups; i++) + printk(" %Ld", (long long)fs->group_info[i].freespace); + printk("\nzones: %Ld %Ld %Ld\n", + (long long)fs->zones[0].freespace, + (long long)fs->zones[1].freespace, + (long long)fs->zones[2].freespace); + }*/ + spadfs_free_super(s); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,37) +__cold static struct dentry *spadfs_mount(struct file_system_type *fs_type, int flags, + const char *dev_name, void *data) +{ + return mount_bdev(fs_type, flags, dev_name, data, spadfs_fill_super); +} +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) +__cold static int spadfs_get_sb(struct file_system_type *fs_type, int flags, + const char *dev_name, void *data, struct vfsmount *mnt) +{ + return get_sb_bdev(fs_type, flags, dev_name, data, spadfs_fill_super, + mnt); +} +#else +__cold static struct super_block *spadfs_get_sb(struct file_system_type *fs_type, + int flags, const char *dev_name, + void *data) +{ + return get_sb_bdev(fs_type, flags, dev_name, data, spadfs_fill_super); +} +#endif + +static +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21) +const +#endif +struct super_operations spadfs_sops = { + .alloc_inode = spadfs_alloc_inode, + .destroy_inode = spadfs_destroy_inode, +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) + .delete_inode = spadfs_delete_inode, +#else + .evict_inode = spadfs_evict_inode, +#endif + .put_super = spadfs_put_super, + .sync_fs = spadfs_sync_fs, + .statfs = spadfs_statfs, + .remount_fs = spadfs_remount_fs, + .show_options = spadfs_show_options, +#ifdef SPADFS_QUOTA + .quota_read = spadfs_quota_read, + .quota_write = spadfs_quota_write, +#if SPADFS_QUOTA >= 2 + .get_dquots = spadfs_quota_get, +#endif +#endif +}; + +static struct file_system_type spadfs_fs_type = { + .owner = THIS_MODULE, + .name = __stringify(SPADFS_NAME), +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,37) + .mount = spadfs_mount, +#else + .get_sb = spadfs_get_sb, +#endif + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV +#if TEST_RHEL_VERSION(5,3) + | FS_HAS_FALLOCATE +#endif +#if TEST_RHEL_VERSION(5,4) + | FS_HAS_FIEMAP +#endif +}; + +#ifdef CONFIG_SMP +unsigned spadfs_nr_cpus; +int spadfs_unlocked_extent_cache; +unsigned spadfs_extent_cache_size; +#endif + +__cold static void spadfs_free_caches(void) +{ + rcu_barrier(); + spadfs_free_cache(spadfs_inode_cachep); + spadfs_free_cache(spadfs_ea_cachep); + spadfs_free_cache(spadfs_extent_cachep); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,31) +static int minimize_inode_size = 0; +#else +static bool minimize_inode_size = 0; +#endif +module_param(minimize_inode_size, bool, S_IRUGO); + +__cold static int __init spadfs_init(void) +{ + int r; + + if (!spadfs_struct_check_correctness()) { + printk(KERN_ERR "SPADFS: BAD STRUCTURE ALIGNMENT, THE PROGRAM WAS COMPILED WITH WRONG ABI\n"); + r = -EINVAL; + goto ret_r; + } + +#ifdef CONFIG_SMP +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0) + { + unsigned i; + spadfs_nr_cpus = 1; + for (i = 0; i < nr_cpumask_bits; i++) + if (cpu_possible(i)) + spadfs_nr_cpus = i + 1; + } +#else + spadfs_nr_cpus = cpumask_last(cpu_possible_mask) + 1; +#endif + + spadfs_unlocked_extent_cache = sizeof(SPADFNODE) + spadfs_nr_cpus * sizeof(struct extent_cache) <= PAGE_SIZE; + if (spadfs_nr_cpus > 1 && minimize_inode_size) + spadfs_unlocked_extent_cache = 0; + spadfs_extent_cache_size = spadfs_unlocked_extent_cache ? spadfs_nr_cpus : 1; +#endif + + spadfs_inode_cachep = kmem_cache_create(__stringify(SPADFS_NAME) "_inode_cache", + sizeof(SPADFNODE) + spadfs_extent_cache_size * sizeof(struct extent_cache), + FNODE_EA_ALIGN, SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, + fnode_ctor +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) + , NULL +#endif + ); + if (unlikely(!spadfs_inode_cachep)) { + r = -ENOMEM; + goto ret_r; + } + + spadfs_ea_cachep = kmem_cache_create(__stringify(SPADFS_NAME) "_ea_cache", + FNODE_MAX_EA_SIZE, + FNODE_EA_ALIGN, SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, + NULL +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) + , NULL +#endif + ); + if (unlikely(!spadfs_ea_cachep)) { + r = -ENOMEM; + goto ret_r; + } + + spadfs_extent_cachep = kmem_cache_create(__stringify(SPADFS_NAME) "_extent_cache", + sizeof(struct allocmem), + 0, SLAB_MEM_SPREAD, + NULL +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) + , NULL +#endif + ); + if (unlikely(!spadfs_extent_cachep)) { + r = -ENOMEM; + goto ret_r; + } + + r = register_filesystem(&spadfs_fs_type); + if (unlikely(r)) + goto ret_r; + + return 0; + +ret_r: + spadfs_free_caches(); + return r; +} + +__cold static void __exit spadfs_exit(void) +{ + unregister_filesystem(&spadfs_fs_type); + spadfs_free_caches(); +} + +module_init(spadfs_init) +module_exit(spadfs_exit) +MODULE_LICENSE("GPL"); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0) +MODULE_ALIAS_FS(__stringify(SPADFS_NAME)); +#endif + +#if __GNUC__ <= 3 && BITS_PER_LONG == 32 +/* + * GCC 2.95.* has a bug --- it generates ".globl __udivdi3" into assembler + * although it doesn't ever call this function. This causes module linking + * problems. + * + * GCC 3.3.6 does it too. I have no idea why. + */ +void __udivdi3(void) +{ + printk(KERN_EMERG "__udivdi3 called!\n"); + BUG(); +} +#endif diff --git a/fs/spadfs/xattr.c b/fs/spadfs/xattr.c new file mode 100644 index 000000000..d0b7345f0 --- /dev/null +++ b/fs/spadfs/xattr.c @@ -0,0 +1,352 @@ +#include "spadfs.h" + +#ifdef SPADFS_XATTR + +static int spadfs_xattr_get( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,0) + const struct xattr_handler *handler, +#endif + struct dentry *dentry, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,7,0) + struct inode *inode, +#endif + const char *name, + void *buffer, size_t buffer_size +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,4,0) + , int type +#endif + ) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) + SPADFNODE *f = spadfnode(dentry->d_inode); +#else + SPADFNODE *f = spadfnode(inode); +#endif + SPADFS *fs = f->fs; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,0) + int type = handler->flags; +#endif + + int r; + + struct fnode_ea *xat; + unsigned xat_size; + + u8 *found; + + sync_lock_decl + + down_read_sync_lock(fs); + + xat = spadfs_get_ea(f, EA_XATTR_MAGIC, EA_XATTR_MAGIC_MASK); + if (!xat) { + r = -ENODATA; + goto ret_r; + } + if (unlikely(IS_ERR(xat))) { + r = PTR_ERR(xat); + goto ret_r; + } + + xat_size = (SPAD2CPU32_LV(&xat->magic) >> FNODE_EA_SIZE_SHIFT) & + FNODE_EA_SIZE_MASK_1; + found = GET_XAT((u8 *)(xat + 1), xat_size, GET_XAT_TYPE_NAME, + type, name, strlen(name)); + if (unlikely(found == GET_XAT_ERROR)) { + spadfs_error(fs, TXFLAGS_EA_ERROR, "XAT extended attribute error on fnode %Lx/%x", + (unsigned long long)f->fnode_block, f->fnode_pos); + r = -EFSERROR; + goto ret_r; + } + if (!found) { + r = -ENODATA; + goto ret_r; + } + if (buffer) { + if (unlikely(found[2] > buffer_size)) { + r = -ERANGE; + goto ret_r; + } + memcpy(buffer, found + 3 + found[1], found[2]); + } + r = found[2]; + +ret_r: + up_read_sync_lock(fs); + + return r; +#undef fs +} + +static int spadfs_xattr_set( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,0) + const struct xattr_handler *handler, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) + struct mnt_idmap *ns, +#endif + struct dentry *dentry, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,7,0) + struct inode *inode, +#endif + const char *name, + const void *value, size_t valuelen, int flags +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,4,0) + , int type +#endif + ) +{ + SPADFNODE *f = spadfnode(dentry->d_inode); + SPADFS *fs = f->fs; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,0) + int type = handler->flags; +#endif + + unsigned namelen = strlen(name); + + u8 *ea; + unsigned ea_size; + int r; + + struct fnode_ea *xat; + unsigned xat_size; + + u8 *found; + + ea = kmalloc(FNODE_MAX_EA_SIZE, GFP_NOIO); + if (unlikely(!ea)) + return -ENOMEM; + + down_write_sync_lock(fs); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,7,0) + if (unlikely(dentry->d_inode != inode)) { + r = -EINVAL; + goto ret_r; + } +#endif + + /* ext2 has these tests for NULL too, I don't know why */ + if (unlikely(!name)) { + r = -EINVAL; + goto ret_r; + } + if (unlikely(!value)) + valuelen = 0; + + if (unlikely(namelen > 0xff) || unlikely(valuelen > 0xff)) { + r = -ERANGE; + goto ret_r; + } + + if (unlikely(!namelen)) { + r = -EINVAL; + goto ret_r; + } + + ea_size = f->ea_size; + memcpy(ea, f->ea, ea_size); + + xat = GET_EA((struct fnode_ea *)ea, ea_size, EA_XATTR_MAGIC, EA_XATTR_MAGIC_MASK); + if (unlikely(xat == GET_EA_ERROR)) { + spadfs_error(fs, TXFLAGS_FS_ERROR, + "error parsing extended attributes on fnode %Lx/%x", + (unsigned long long)f->fnode_block, f->fnode_pos); + r = -EFSERROR; + goto ret_r; + } + if (!xat) { + const unsigned new_ea_size = FNODE_EA_DO_ALIGN(sizeof(struct fnode_ea)); + if (unlikely(ea_size + new_ea_size > FNODE_MAX_EA_SIZE)) { + r = -ENOSPC; + goto ret_r; + } + memset(ea + ea_size, 0, new_ea_size); + xat = (struct fnode_ea *)(ea + ea_size); + CPU2SPAD32_LV(&xat->magic, EA_XATTR_MAGIC); + ea_size += new_ea_size; + } + + xat_size = (SPAD2CPU32_LV(&xat->magic) >> FNODE_EA_SIZE_SHIFT) & + FNODE_EA_SIZE_MASK_1; + + found = GET_XAT((u8 *)(xat + 1), xat_size, GET_XAT_TYPE_NAME, + type, name, namelen); + if (unlikely(found == GET_XAT_ERROR)) { + spadfs_error(fs, TXFLAGS_EA_ERROR, "XAT extended attribute error on fnode %Lx/%x", + (unsigned long long)f->fnode_block, f->fnode_pos); + r = -EFSERROR; + goto ret_r; + } + if (found) { + u8 *end; + unsigned shrink; + if (unlikely(flags & XATTR_CREATE)) { + r = -EEXIST; + goto ret_r; + } + if (valuelen == found[2]) + goto set_just_value; + end = (u8 *)(xat + 1) + xat_size; + shrink = 3 + found[1] + found[2]; + memmove(found, found + shrink, end - (found + shrink)); + r = RESIZE_EA(ea, &ea_size, xat, xat_size - shrink); + if (unlikely(r)) { + goto ret_r; + } + xat_size -= shrink; + if (!valuelen) { + if (!xat_size) { + REMOVE_EA(ea, &ea_size, xat); + } + goto refile_ret; + } + goto add_new_key; + } else { + if (unlikely(flags & XATTR_REPLACE)) { + r = -ENODATA; + goto ret_r; + } + if (!valuelen) { + r = 0; + goto ret_r; + } +add_new_key: + r = RESIZE_EA(ea, &ea_size, xat, xat_size + 3 + namelen + valuelen); + if (unlikely(r)) { + goto ret_r; + } + found = (u8 *)(xat + 1) + xat_size; + } + + found[0] = type; + found[1] = namelen; + found[2] = valuelen; + memcpy(found + 3, name, namelen); +set_just_value: + memcpy(found + 3 + namelen, value, valuelen); + +refile_ret: + r = spadfs_refile_fnode(spadfnode(dentry->d_parent->d_inode), &dentry->d_name, f, ea, ea_size); + +ret_r: + up_write_sync_lock(fs); + kfree(ea); + + return r; +#undef fs +} + +ssize_t spadfs_listxattr(struct dentry *dentry, char *list, size_t list_size) +{ + SPADFNODE *f = spadfnode(dentry->d_inode); + SPADFS *fs = f->fs; + + int r; + + struct fnode_ea *xat; + + u8 *found, *end; + unsigned xat_size; + + sync_lock_decl + + down_read_sync_lock(fs); + + xat = spadfs_get_ea(f, EA_XATTR_MAGIC, EA_XATTR_MAGIC_MASK); + if (!xat) { + r = 0; + goto ret_r; + } + if (unlikely(IS_ERR(xat))) { + r = PTR_ERR(xat); + goto ret_r; + } + + xat_size = (SPAD2CPU32_LV(&xat->magic) >> FNODE_EA_SIZE_SHIFT) & + FNODE_EA_SIZE_MASK_1; + found = (u8 *)(xat + 1); + end = found + xat_size; + r = 0; + while (1) { + unsigned len, prefixlen; + const struct xattr_handler **handler_p; + + found = GET_XAT(found, end - found, GET_XAT_ALL, 0, NULL, 0); + if (unlikely(found == GET_XAT_ERROR)) { + spadfs_error(fs, TXFLAGS_EA_ERROR, "XAT extended attribute error on fnode %Lx/%x", + (unsigned long long)f->fnode_block, f->fnode_pos); + r = -EFSERROR; + goto ret_r; + } + if (!found) + break; + + if (found[0] == SPADFS_XATTR_TRUSTED && !capable(CAP_SYS_ADMIN)) + goto skip_this; + + for (handler_p = spadfs_xattr_handlers; *handler_p; handler_p++) + if ((*handler_p)->flags == found[0]) + goto found_handler; + goto skip_this; +found_handler: + + prefixlen = strlen((*handler_p)->prefix); + + len = prefixlen + found[1] + 1; + + if (!list) { + r += len; + } else { + if (list_size < len) { + r = -ERANGE; + goto ret_r; + } + memcpy(list, (*handler_p)->prefix, prefixlen); + memcpy(list + prefixlen, found + 3, found[1]); + list[prefixlen + found[1]] = 0; + list_size -= len; + list += len; + r += len; + } + +skip_this: + found += 3 + found[1] + found[2]; + } + +ret_r: + up_read_sync_lock(fs); + + return r; +#undef fs +} + +static const struct xattr_handler spadfs_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .get = spadfs_xattr_get, + .set = spadfs_xattr_set, + .flags = SPADFS_XATTR_USER, +}; + +static const struct xattr_handler spadfs_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = spadfs_xattr_get, + .set = spadfs_xattr_set, + .flags = SPADFS_XATTR_SECURITY, +}; + +static const struct xattr_handler spadfs_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = spadfs_xattr_get, + .set = spadfs_xattr_set, + .flags = SPADFS_XATTR_TRUSTED, +}; + +const struct xattr_handler *spadfs_xattr_handlers[] = { + &spadfs_xattr_user_handler, + &spadfs_xattr_security_handler, + &spadfs_xattr_trusted_handler, + NULL +}; + +#endif -- 2.42.0