|
@@ -53,9 +53,9 @@ static ssize_t name##_show(struct device *d, \
|
|
|
} \
|
|
|
static DEVICE_ATTR_RO(name);
|
|
|
|
|
|
-static inline int init_done(struct zram *zram)
|
|
|
+static inline bool init_done(struct zram *zram)
|
|
|
{
|
|
|
- return zram->meta != NULL;
|
|
|
+ return zram->disksize;
|
|
|
}
|
|
|
|
|
|
static inline struct zram *dev_to_zram(struct device *dev)
|
|
@@ -307,42 +307,67 @@ static inline int valid_io_request(struct zram *zram,
|
|
|
return 1;
|
|
|
}
|
|
|
|
|
|
-static void zram_meta_free(struct zram_meta *meta)
|
|
|
+static void zram_meta_free(struct zram_meta *meta, u64 disksize)
|
|
|
{
|
|
|
+ size_t num_pages = disksize >> PAGE_SHIFT;
|
|
|
+ size_t index;
|
|
|
+
|
|
|
+ /* Free all pages that are still in this zram device */
|
|
|
+ for (index = 0; index < num_pages; index++) {
|
|
|
+ unsigned long handle = meta->table[index].handle;
|
|
|
+
|
|
|
+ if (!handle)
|
|
|
+ continue;
|
|
|
+
|
|
|
+ zs_free(meta->mem_pool, handle);
|
|
|
+ }
|
|
|
+
|
|
|
zs_destroy_pool(meta->mem_pool);
|
|
|
vfree(meta->table);
|
|
|
kfree(meta);
|
|
|
}
|
|
|
|
|
|
-static struct zram_meta *zram_meta_alloc(u64 disksize)
|
|
|
+static struct zram_meta *zram_meta_alloc(int device_id, u64 disksize)
|
|
|
{
|
|
|
size_t num_pages;
|
|
|
+ char pool_name[8];
|
|
|
struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
|
|
|
+
|
|
|
if (!meta)
|
|
|
- goto out;
|
|
|
+ return NULL;
|
|
|
|
|
|
num_pages = disksize >> PAGE_SHIFT;
|
|
|
meta->table = vzalloc(num_pages * sizeof(*meta->table));
|
|
|
if (!meta->table) {
|
|
|
pr_err("Error allocating zram address table\n");
|
|
|
- goto free_meta;
|
|
|
+ goto out_error;
|
|
|
}
|
|
|
|
|
|
- meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
|
|
|
+ snprintf(pool_name, sizeof(pool_name), "zram%d", device_id);
|
|
|
+ meta->mem_pool = zs_create_pool(pool_name, GFP_NOIO | __GFP_HIGHMEM);
|
|
|
if (!meta->mem_pool) {
|
|
|
pr_err("Error creating memory pool\n");
|
|
|
- goto free_table;
|
|
|
+ goto out_error;
|
|
|
}
|
|
|
|
|
|
return meta;
|
|
|
|
|
|
-free_table:
|
|
|
+out_error:
|
|
|
vfree(meta->table);
|
|
|
-free_meta:
|
|
|
kfree(meta);
|
|
|
- meta = NULL;
|
|
|
-out:
|
|
|
- return meta;
|
|
|
+ return NULL;
|
|
|
+}
|
|
|
+
|
|
|
+static inline bool zram_meta_get(struct zram *zram)
|
|
|
+{
|
|
|
+ if (atomic_inc_not_zero(&zram->refcount))
|
|
|
+ return true;
|
|
|
+ return false;
|
|
|
+}
|
|
|
+
|
|
|
+static inline void zram_meta_put(struct zram *zram)
|
|
|
+{
|
|
|
+ atomic_dec(&zram->refcount);
|
|
|
}
|
|
|
|
|
|
static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
|
|
@@ -704,10 +729,11 @@ static void zram_bio_discard(struct zram *zram, u32 index,
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-static void zram_reset_device(struct zram *zram, bool reset_capacity)
|
|
|
+static void zram_reset_device(struct zram *zram)
|
|
|
{
|
|
|
- size_t index;
|
|
|
struct zram_meta *meta;
|
|
|
+ struct zcomp *comp;
|
|
|
+ u64 disksize;
|
|
|
|
|
|
down_write(&zram->init_lock);
|
|
|
|
|
@@ -719,36 +745,30 @@ static void zram_reset_device(struct zram *zram, bool reset_capacity)
|
|
|
}
|
|
|
|
|
|
meta = zram->meta;
|
|
|
- /* Free all pages that are still in this zram device */
|
|
|
- for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
|
|
|
- unsigned long handle = meta->table[index].handle;
|
|
|
- if (!handle)
|
|
|
- continue;
|
|
|
-
|
|
|
- zs_free(meta->mem_pool, handle);
|
|
|
- }
|
|
|
-
|
|
|
- zcomp_destroy(zram->comp);
|
|
|
- zram->max_comp_streams = 1;
|
|
|
+ comp = zram->comp;
|
|
|
+ disksize = zram->disksize;
|
|
|
+ /*
|
|
|
+ * Refcount will go down to 0 eventually and r/w handler
|
|
|
+ * cannot handle further I/O so it will bail out by
|
|
|
+ * check zram_meta_get.
|
|
|
+ */
|
|
|
+ zram_meta_put(zram);
|
|
|
+ /*
|
|
|
+ * We want to free zram_meta in process context to avoid
|
|
|
+ * deadlock between reclaim path and any other locks.
|
|
|
+ */
|
|
|
+ wait_event(zram->io_done, atomic_read(&zram->refcount) == 0);
|
|
|
|
|
|
- zram_meta_free(zram->meta);
|
|
|
- zram->meta = NULL;
|
|
|
/* Reset stats */
|
|
|
memset(&zram->stats, 0, sizeof(zram->stats));
|
|
|
-
|
|
|
zram->disksize = 0;
|
|
|
- if (reset_capacity)
|
|
|
- set_capacity(zram->disk, 0);
|
|
|
+ zram->max_comp_streams = 1;
|
|
|
+ set_capacity(zram->disk, 0);
|
|
|
|
|
|
up_write(&zram->init_lock);
|
|
|
-
|
|
|
- /*
|
|
|
- * Revalidate disk out of the init_lock to avoid lockdep splat.
|
|
|
- * It's okay because disk's capacity is protected by init_lock
|
|
|
- * so that revalidate_disk always sees up-to-date capacity.
|
|
|
- */
|
|
|
- if (reset_capacity)
|
|
|
- revalidate_disk(zram->disk);
|
|
|
+ /* I/O operation under all of CPU are done so let's free */
|
|
|
+ zram_meta_free(meta, disksize);
|
|
|
+ zcomp_destroy(comp);
|
|
|
}
|
|
|
|
|
|
static ssize_t disksize_store(struct device *dev,
|
|
@@ -765,7 +785,7 @@ static ssize_t disksize_store(struct device *dev,
|
|
|
return -EINVAL;
|
|
|
|
|
|
disksize = PAGE_ALIGN(disksize);
|
|
|
- meta = zram_meta_alloc(disksize);
|
|
|
+ meta = zram_meta_alloc(zram->disk->first_minor, disksize);
|
|
|
if (!meta)
|
|
|
return -ENOMEM;
|
|
|
|
|
@@ -784,6 +804,8 @@ static ssize_t disksize_store(struct device *dev,
|
|
|
goto out_destroy_comp;
|
|
|
}
|
|
|
|
|
|
+ init_waitqueue_head(&zram->io_done);
|
|
|
+ atomic_set(&zram->refcount, 1);
|
|
|
zram->meta = meta;
|
|
|
zram->comp = comp;
|
|
|
zram->disksize = disksize;
|
|
@@ -803,7 +825,7 @@ out_destroy_comp:
|
|
|
up_write(&zram->init_lock);
|
|
|
zcomp_destroy(comp);
|
|
|
out_free_meta:
|
|
|
- zram_meta_free(meta);
|
|
|
+ zram_meta_free(meta, disksize);
|
|
|
return err;
|
|
|
}
|
|
|
|
|
@@ -821,8 +843,9 @@ static ssize_t reset_store(struct device *dev,
|
|
|
if (!bdev)
|
|
|
return -ENOMEM;
|
|
|
|
|
|
+ mutex_lock(&bdev->bd_mutex);
|
|
|
/* Do not reset an active device! */
|
|
|
- if (bdev->bd_holders) {
|
|
|
+ if (bdev->bd_openers) {
|
|
|
ret = -EBUSY;
|
|
|
goto out;
|
|
|
}
|
|
@@ -838,12 +861,16 @@ static ssize_t reset_store(struct device *dev,
|
|
|
|
|
|
/* Make sure all pending I/O is finished */
|
|
|
fsync_bdev(bdev);
|
|
|
+ zram_reset_device(zram);
|
|
|
+
|
|
|
+ mutex_unlock(&bdev->bd_mutex);
|
|
|
+ revalidate_disk(zram->disk);
|
|
|
bdput(bdev);
|
|
|
|
|
|
- zram_reset_device(zram, true);
|
|
|
return len;
|
|
|
|
|
|
out:
|
|
|
+ mutex_unlock(&bdev->bd_mutex);
|
|
|
bdput(bdev);
|
|
|
return ret;
|
|
|
}
|
|
@@ -909,23 +936,21 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio)
|
|
|
{
|
|
|
struct zram *zram = queue->queuedata;
|
|
|
|
|
|
- down_read(&zram->init_lock);
|
|
|
- if (unlikely(!init_done(zram)))
|
|
|
+ if (unlikely(!zram_meta_get(zram)))
|
|
|
goto error;
|
|
|
|
|
|
if (!valid_io_request(zram, bio->bi_iter.bi_sector,
|
|
|
bio->bi_iter.bi_size)) {
|
|
|
atomic64_inc(&zram->stats.invalid_io);
|
|
|
- goto error;
|
|
|
+ goto put_zram;
|
|
|
}
|
|
|
|
|
|
__zram_make_request(zram, bio);
|
|
|
- up_read(&zram->init_lock);
|
|
|
-
|
|
|
+ zram_meta_put(zram);
|
|
|
return;
|
|
|
-
|
|
|
+put_zram:
|
|
|
+ zram_meta_put(zram);
|
|
|
error:
|
|
|
- up_read(&zram->init_lock);
|
|
|
bio_io_error(bio);
|
|
|
}
|
|
|
|
|
@@ -947,21 +972,19 @@ static void zram_slot_free_notify(struct block_device *bdev,
|
|
|
static int zram_rw_page(struct block_device *bdev, sector_t sector,
|
|
|
struct page *page, int rw)
|
|
|
{
|
|
|
- int offset, err;
|
|
|
+ int offset, err = -EIO;
|
|
|
u32 index;
|
|
|
struct zram *zram;
|
|
|
struct bio_vec bv;
|
|
|
|
|
|
zram = bdev->bd_disk->private_data;
|
|
|
+ if (unlikely(!zram_meta_get(zram)))
|
|
|
+ goto out;
|
|
|
+
|
|
|
if (!valid_io_request(zram, sector, PAGE_SIZE)) {
|
|
|
atomic64_inc(&zram->stats.invalid_io);
|
|
|
- return -EINVAL;
|
|
|
- }
|
|
|
-
|
|
|
- down_read(&zram->init_lock);
|
|
|
- if (unlikely(!init_done(zram))) {
|
|
|
- err = -EIO;
|
|
|
- goto out_unlock;
|
|
|
+ err = -EINVAL;
|
|
|
+ goto put_zram;
|
|
|
}
|
|
|
|
|
|
index = sector >> SECTORS_PER_PAGE_SHIFT;
|
|
@@ -972,8 +995,9 @@ static int zram_rw_page(struct block_device *bdev, sector_t sector,
|
|
|
bv.bv_offset = 0;
|
|
|
|
|
|
err = zram_bvec_rw(zram, &bv, index, offset, rw);
|
|
|
-out_unlock:
|
|
|
- up_read(&zram->init_lock);
|
|
|
+put_zram:
|
|
|
+ zram_meta_put(zram);
|
|
|
+out:
|
|
|
/*
|
|
|
* If I/O fails, just return error(ie, non-zero) without
|
|
|
* calling page_endio.
|
|
@@ -1039,19 +1063,19 @@ static struct attribute_group zram_disk_attr_group = {
|
|
|
|
|
|
static int create_device(struct zram *zram, int device_id)
|
|
|
{
|
|
|
+ struct request_queue *queue;
|
|
|
int ret = -ENOMEM;
|
|
|
|
|
|
init_rwsem(&zram->init_lock);
|
|
|
|
|
|
- zram->queue = blk_alloc_queue(GFP_KERNEL);
|
|
|
- if (!zram->queue) {
|
|
|
+ queue = blk_alloc_queue(GFP_KERNEL);
|
|
|
+ if (!queue) {
|
|
|
pr_err("Error allocating disk queue for device %d\n",
|
|
|
device_id);
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
- blk_queue_make_request(zram->queue, zram_make_request);
|
|
|
- zram->queue->queuedata = zram;
|
|
|
+ blk_queue_make_request(queue, zram_make_request);
|
|
|
|
|
|
/* gendisk structure */
|
|
|
zram->disk = alloc_disk(1);
|
|
@@ -1064,7 +1088,8 @@ static int create_device(struct zram *zram, int device_id)
|
|
|
zram->disk->major = zram_major;
|
|
|
zram->disk->first_minor = device_id;
|
|
|
zram->disk->fops = &zram_devops;
|
|
|
- zram->disk->queue = zram->queue;
|
|
|
+ zram->disk->queue = queue;
|
|
|
+ zram->disk->queue->queuedata = zram;
|
|
|
zram->disk->private_data = zram;
|
|
|
snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
|
|
|
|
|
@@ -1115,20 +1140,35 @@ out_free_disk:
|
|
|
del_gendisk(zram->disk);
|
|
|
put_disk(zram->disk);
|
|
|
out_free_queue:
|
|
|
- blk_cleanup_queue(zram->queue);
|
|
|
+ blk_cleanup_queue(queue);
|
|
|
out:
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
-static void destroy_device(struct zram *zram)
|
|
|
+static void destroy_devices(unsigned int nr)
|
|
|
{
|
|
|
- sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
|
|
|
- &zram_disk_attr_group);
|
|
|
+ struct zram *zram;
|
|
|
+ unsigned int i;
|
|
|
|
|
|
- del_gendisk(zram->disk);
|
|
|
- put_disk(zram->disk);
|
|
|
+ for (i = 0; i < nr; i++) {
|
|
|
+ zram = &zram_devices[i];
|
|
|
+ /*
|
|
|
+ * Remove sysfs first, so no one will perform a disksize
|
|
|
+ * store while we destroy the devices
|
|
|
+ */
|
|
|
+ sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
|
|
|
+ &zram_disk_attr_group);
|
|
|
+
|
|
|
+ zram_reset_device(zram);
|
|
|
|
|
|
- blk_cleanup_queue(zram->queue);
|
|
|
+ blk_cleanup_queue(zram->disk->queue);
|
|
|
+ del_gendisk(zram->disk);
|
|
|
+ put_disk(zram->disk);
|
|
|
+ }
|
|
|
+
|
|
|
+ kfree(zram_devices);
|
|
|
+ unregister_blkdev(zram_major, "zram");
|
|
|
+ pr_info("Destroyed %u device(s)\n", nr);
|
|
|
}
|
|
|
|
|
|
static int __init zram_init(void)
|
|
@@ -1138,64 +1178,39 @@ static int __init zram_init(void)
|
|
|
if (num_devices > max_num_devices) {
|
|
|
pr_warn("Invalid value for num_devices: %u\n",
|
|
|
num_devices);
|
|
|
- ret = -EINVAL;
|
|
|
- goto out;
|
|
|
+ return -EINVAL;
|
|
|
}
|
|
|
|
|
|
zram_major = register_blkdev(0, "zram");
|
|
|
if (zram_major <= 0) {
|
|
|
pr_warn("Unable to get major number\n");
|
|
|
- ret = -EBUSY;
|
|
|
- goto out;
|
|
|
+ return -EBUSY;
|
|
|
}
|
|
|
|
|
|
/* Allocate the device array and initialize each one */
|
|
|
zram_devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);
|
|
|
if (!zram_devices) {
|
|
|
- ret = -ENOMEM;
|
|
|
- goto unregister;
|
|
|
+ unregister_blkdev(zram_major, "zram");
|
|
|
+ return -ENOMEM;
|
|
|
}
|
|
|
|
|
|
for (dev_id = 0; dev_id < num_devices; dev_id++) {
|
|
|
ret = create_device(&zram_devices[dev_id], dev_id);
|
|
|
if (ret)
|
|
|
- goto free_devices;
|
|
|
+ goto out_error;
|
|
|
}
|
|
|
|
|
|
- pr_info("Created %u device(s) ...\n", num_devices);
|
|
|
-
|
|
|
+ pr_info("Created %u device(s)\n", num_devices);
|
|
|
return 0;
|
|
|
|
|
|
-free_devices:
|
|
|
- while (dev_id)
|
|
|
- destroy_device(&zram_devices[--dev_id]);
|
|
|
- kfree(zram_devices);
|
|
|
-unregister:
|
|
|
- unregister_blkdev(zram_major, "zram");
|
|
|
-out:
|
|
|
+out_error:
|
|
|
+ destroy_devices(dev_id);
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
static void __exit zram_exit(void)
|
|
|
{
|
|
|
- int i;
|
|
|
- struct zram *zram;
|
|
|
-
|
|
|
- for (i = 0; i < num_devices; i++) {
|
|
|
- zram = &zram_devices[i];
|
|
|
-
|
|
|
- destroy_device(zram);
|
|
|
- /*
|
|
|
- * Shouldn't access zram->disk after destroy_device
|
|
|
- * because destroy_device already released zram->disk.
|
|
|
- */
|
|
|
- zram_reset_device(zram, false);
|
|
|
- }
|
|
|
-
|
|
|
- unregister_blkdev(zram_major, "zram");
|
|
|
-
|
|
|
- kfree(zram_devices);
|
|
|
- pr_debug("Cleanup done!\n");
|
|
|
+ destroy_devices(num_devices);
|
|
|
}
|
|
|
|
|
|
module_init(zram_init);
|