|
@@ -29,19 +29,36 @@
|
|
|
#include <linux/genhd.h>
|
|
|
#include <linux/highmem.h>
|
|
|
#include <linux/slab.h>
|
|
|
-#include <linux/lzo.h>
|
|
|
#include <linux/string.h>
|
|
|
#include <linux/vmalloc.h>
|
|
|
+#include <linux/err.h>
|
|
|
|
|
|
#include "zram_drv.h"
|
|
|
|
|
|
/* Globals */
|
|
|
static int zram_major;
|
|
|
static struct zram *zram_devices;
|
|
|
+static const char *default_compressor = "lzo";
|
|
|
|
|
|
/* Module params (documentation at end) */
|
|
|
static unsigned int num_devices = 1;
|
|
|
|
|
|
+#define ZRAM_ATTR_RO(name) \
|
|
|
+static ssize_t zram_attr_##name##_show(struct device *d, \
|
|
|
+ struct device_attribute *attr, char *b) \
|
|
|
+{ \
|
|
|
+ struct zram *zram = dev_to_zram(d); \
|
|
|
+ return scnprintf(b, PAGE_SIZE, "%llu\n", \
|
|
|
+ (u64)atomic64_read(&zram->stats.name)); \
|
|
|
+} \
|
|
|
+static struct device_attribute dev_attr_##name = \
|
|
|
+ __ATTR(name, S_IRUGO, zram_attr_##name##_show, NULL);
|
|
|
+
|
|
|
+static inline int init_done(struct zram *zram)
|
|
|
+{
|
|
|
+ return zram->meta != NULL;
|
|
|
+}
|
|
|
+
|
|
|
static inline struct zram *dev_to_zram(struct device *dev)
|
|
|
{
|
|
|
return (struct zram *)dev_to_disk(dev)->private_data;
|
|
@@ -52,92 +69,114 @@ static ssize_t disksize_show(struct device *dev,
|
|
|
{
|
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
|
|
|
- return sprintf(buf, "%llu\n", zram->disksize);
|
|
|
+ return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
|
|
|
}
|
|
|
|
|
|
static ssize_t initstate_show(struct device *dev,
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
{
|
|
|
+ u32 val;
|
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
|
|
|
- return sprintf(buf, "%u\n", zram->init_done);
|
|
|
-}
|
|
|
-
|
|
|
-static ssize_t num_reads_show(struct device *dev,
|
|
|
- struct device_attribute *attr, char *buf)
|
|
|
-{
|
|
|
- struct zram *zram = dev_to_zram(dev);
|
|
|
+ down_read(&zram->init_lock);
|
|
|
+ val = init_done(zram);
|
|
|
+ up_read(&zram->init_lock);
|
|
|
|
|
|
- return sprintf(buf, "%llu\n",
|
|
|
- (u64)atomic64_read(&zram->stats.num_reads));
|
|
|
+ return scnprintf(buf, PAGE_SIZE, "%u\n", val);
|
|
|
}
|
|
|
|
|
|
-static ssize_t num_writes_show(struct device *dev,
|
|
|
+static ssize_t orig_data_size_show(struct device *dev,
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
{
|
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
|
|
|
- return sprintf(buf, "%llu\n",
|
|
|
- (u64)atomic64_read(&zram->stats.num_writes));
|
|
|
+ return scnprintf(buf, PAGE_SIZE, "%llu\n",
|
|
|
+ (u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
|
|
|
}
|
|
|
|
|
|
-static ssize_t invalid_io_show(struct device *dev,
|
|
|
+static ssize_t mem_used_total_show(struct device *dev,
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
{
|
|
|
+ u64 val = 0;
|
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
+ struct zram_meta *meta = zram->meta;
|
|
|
|
|
|
- return sprintf(buf, "%llu\n",
|
|
|
- (u64)atomic64_read(&zram->stats.invalid_io));
|
|
|
-}
|
|
|
-
|
|
|
-static ssize_t notify_free_show(struct device *dev,
|
|
|
- struct device_attribute *attr, char *buf)
|
|
|
-{
|
|
|
- struct zram *zram = dev_to_zram(dev);
|
|
|
+ down_read(&zram->init_lock);
|
|
|
+ if (init_done(zram))
|
|
|
+ val = zs_get_total_size_bytes(meta->mem_pool);
|
|
|
+ up_read(&zram->init_lock);
|
|
|
|
|
|
- return sprintf(buf, "%llu\n",
|
|
|
- (u64)atomic64_read(&zram->stats.notify_free));
|
|
|
+ return scnprintf(buf, PAGE_SIZE, "%llu\n", val);
|
|
|
}
|
|
|
|
|
|
-static ssize_t zero_pages_show(struct device *dev,
|
|
|
+static ssize_t max_comp_streams_show(struct device *dev,
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
{
|
|
|
+ int val;
|
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
|
|
|
- return sprintf(buf, "%u\n", atomic_read(&zram->stats.pages_zero));
|
|
|
+ down_read(&zram->init_lock);
|
|
|
+ val = zram->max_comp_streams;
|
|
|
+ up_read(&zram->init_lock);
|
|
|
+
|
|
|
+ return scnprintf(buf, PAGE_SIZE, "%d\n", val);
|
|
|
}
|
|
|
|
|
|
-static ssize_t orig_data_size_show(struct device *dev,
|
|
|
- struct device_attribute *attr, char *buf)
|
|
|
+static ssize_t max_comp_streams_store(struct device *dev,
|
|
|
+ struct device_attribute *attr, const char *buf, size_t len)
|
|
|
{
|
|
|
+ int num;
|
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
+ int ret;
|
|
|
|
|
|
- return sprintf(buf, "%llu\n",
|
|
|
- (u64)(atomic_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
|
|
|
-}
|
|
|
+ ret = kstrtoint(buf, 0, &num);
|
|
|
+ if (ret < 0)
|
|
|
+ return ret;
|
|
|
+ if (num < 1)
|
|
|
+ return -EINVAL;
|
|
|
|
|
|
-static ssize_t compr_data_size_show(struct device *dev,
|
|
|
- struct device_attribute *attr, char *buf)
|
|
|
-{
|
|
|
- struct zram *zram = dev_to_zram(dev);
|
|
|
+ down_write(&zram->init_lock);
|
|
|
+ if (init_done(zram)) {
|
|
|
+ if (!zcomp_set_max_streams(zram->comp, num)) {
|
|
|
+ pr_info("Cannot change max compression streams\n");
|
|
|
+ ret = -EINVAL;
|
|
|
+ goto out;
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- return sprintf(buf, "%llu\n",
|
|
|
- (u64)atomic64_read(&zram->stats.compr_size));
|
|
|
+ zram->max_comp_streams = num;
|
|
|
+ ret = len;
|
|
|
+out:
|
|
|
+ up_write(&zram->init_lock);
|
|
|
+ return ret;
|
|
|
}
|
|
|
|
|
|
-static ssize_t mem_used_total_show(struct device *dev,
|
|
|
+static ssize_t comp_algorithm_show(struct device *dev,
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
{
|
|
|
- u64 val = 0;
|
|
|
+ size_t sz;
|
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
- struct zram_meta *meta = zram->meta;
|
|
|
|
|
|
down_read(&zram->init_lock);
|
|
|
- if (zram->init_done)
|
|
|
- val = zs_get_total_size_bytes(meta->mem_pool);
|
|
|
+ sz = zcomp_available_show(zram->compressor, buf);
|
|
|
up_read(&zram->init_lock);
|
|
|
|
|
|
- return sprintf(buf, "%llu\n", val);
|
|
|
+ return sz;
|
|
|
+}
|
|
|
+
|
|
|
+static ssize_t comp_algorithm_store(struct device *dev,
|
|
|
+ struct device_attribute *attr, const char *buf, size_t len)
|
|
|
+{
|
|
|
+ struct zram *zram = dev_to_zram(dev);
|
|
|
+ down_write(&zram->init_lock);
|
|
|
+ if (init_done(zram)) {
|
|
|
+ up_write(&zram->init_lock);
|
|
|
+ pr_info("Can't change algorithm for initialized device\n");
|
|
|
+ return -EBUSY;
|
|
|
+ }
|
|
|
+ strlcpy(zram->compressor, buf, sizeof(zram->compressor));
|
|
|
+ up_write(&zram->init_lock);
|
|
|
+ return len;
|
|
|
}
|
|
|
|
|
|
/* flag operations needs meta->tb_lock */
|
|
@@ -192,8 +231,6 @@ static inline int valid_io_request(struct zram *zram, struct bio *bio)
|
|
|
static void zram_meta_free(struct zram_meta *meta)
|
|
|
{
|
|
|
zs_destroy_pool(meta->mem_pool);
|
|
|
- kfree(meta->compress_workmem);
|
|
|
- free_pages((unsigned long)meta->compress_buffer, 1);
|
|
|
vfree(meta->table);
|
|
|
kfree(meta);
|
|
|
}
|
|
@@ -205,22 +242,11 @@ static struct zram_meta *zram_meta_alloc(u64 disksize)
|
|
|
if (!meta)
|
|
|
goto out;
|
|
|
|
|
|
- meta->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
|
|
|
- if (!meta->compress_workmem)
|
|
|
- goto free_meta;
|
|
|
-
|
|
|
- meta->compress_buffer =
|
|
|
- (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
|
|
|
- if (!meta->compress_buffer) {
|
|
|
- pr_err("Error allocating compressor buffer space\n");
|
|
|
- goto free_workmem;
|
|
|
- }
|
|
|
-
|
|
|
num_pages = disksize >> PAGE_SHIFT;
|
|
|
meta->table = vzalloc(num_pages * sizeof(*meta->table));
|
|
|
if (!meta->table) {
|
|
|
pr_err("Error allocating zram address table\n");
|
|
|
- goto free_buffer;
|
|
|
+ goto free_meta;
|
|
|
}
|
|
|
|
|
|
meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
|
|
@@ -230,15 +256,10 @@ static struct zram_meta *zram_meta_alloc(u64 disksize)
|
|
|
}
|
|
|
|
|
|
rwlock_init(&meta->tb_lock);
|
|
|
- mutex_init(&meta->buffer_lock);
|
|
|
return meta;
|
|
|
|
|
|
free_table:
|
|
|
vfree(meta->table);
|
|
|
-free_buffer:
|
|
|
- free_pages((unsigned long)meta->compress_buffer, 1);
|
|
|
-free_workmem:
|
|
|
- kfree(meta->compress_workmem);
|
|
|
free_meta:
|
|
|
kfree(meta);
|
|
|
meta = NULL;
|
|
@@ -288,7 +309,6 @@ static void zram_free_page(struct zram *zram, size_t index)
|
|
|
{
|
|
|
struct zram_meta *meta = zram->meta;
|
|
|
unsigned long handle = meta->table[index].handle;
|
|
|
- u16 size = meta->table[index].size;
|
|
|
|
|
|
if (unlikely(!handle)) {
|
|
|
/*
|
|
@@ -297,21 +317,15 @@ static void zram_free_page(struct zram *zram, size_t index)
|
|
|
*/
|
|
|
if (zram_test_flag(meta, index, ZRAM_ZERO)) {
|
|
|
zram_clear_flag(meta, index, ZRAM_ZERO);
|
|
|
- atomic_dec(&zram->stats.pages_zero);
|
|
|
+ atomic64_dec(&zram->stats.zero_pages);
|
|
|
}
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
- if (unlikely(size > max_zpage_size))
|
|
|
- atomic_dec(&zram->stats.bad_compress);
|
|
|
-
|
|
|
zs_free(meta->mem_pool, handle);
|
|
|
|
|
|
- if (size <= PAGE_SIZE / 2)
|
|
|
- atomic_dec(&zram->stats.good_compress);
|
|
|
-
|
|
|
- atomic64_sub(meta->table[index].size, &zram->stats.compr_size);
|
|
|
- atomic_dec(&zram->stats.pages_stored);
|
|
|
+ atomic64_sub(meta->table[index].size, &zram->stats.compr_data_size);
|
|
|
+ atomic64_dec(&zram->stats.pages_stored);
|
|
|
|
|
|
meta->table[index].handle = 0;
|
|
|
meta->table[index].size = 0;
|
|
@@ -319,8 +333,7 @@ static void zram_free_page(struct zram *zram, size_t index)
|
|
|
|
|
|
static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
|
|
|
{
|
|
|
- int ret = LZO_E_OK;
|
|
|
- size_t clen = PAGE_SIZE;
|
|
|
+ int ret = 0;
|
|
|
unsigned char *cmem;
|
|
|
struct zram_meta *meta = zram->meta;
|
|
|
unsigned long handle;
|
|
@@ -340,12 +353,12 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
|
|
|
if (size == PAGE_SIZE)
|
|
|
copy_page(mem, cmem);
|
|
|
else
|
|
|
- ret = lzo1x_decompress_safe(cmem, size, mem, &clen);
|
|
|
+ ret = zcomp_decompress(zram->comp, cmem, size, mem);
|
|
|
zs_unmap_object(meta->mem_pool, handle);
|
|
|
read_unlock(&meta->tb_lock);
|
|
|
|
|
|
/* Should NEVER happen. Return bio error if it does. */
|
|
|
- if (unlikely(ret != LZO_E_OK)) {
|
|
|
+ if (unlikely(ret)) {
|
|
|
pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
|
|
|
atomic64_inc(&zram->stats.failed_reads);
|
|
|
return ret;
|
|
@@ -388,7 +401,7 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
|
|
|
|
|
|
ret = zram_decompress_page(zram, uncmem, index);
|
|
|
/* Should NEVER happen. Return bio error if it does. */
|
|
|
- if (unlikely(ret != LZO_E_OK))
|
|
|
+ if (unlikely(ret))
|
|
|
goto out_cleanup;
|
|
|
|
|
|
if (is_partial_io(bvec))
|
|
@@ -413,11 +426,10 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
|
|
|
struct page *page;
|
|
|
unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
|
|
|
struct zram_meta *meta = zram->meta;
|
|
|
+ struct zcomp_strm *zstrm;
|
|
|
bool locked = false;
|
|
|
|
|
|
page = bvec->bv_page;
|
|
|
- src = meta->compress_buffer;
|
|
|
-
|
|
|
if (is_partial_io(bvec)) {
|
|
|
/*
|
|
|
* This is a partial IO. We need to read the full page
|
|
@@ -433,7 +445,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
- mutex_lock(&meta->buffer_lock);
|
|
|
+ zstrm = zcomp_strm_find(zram->comp);
|
|
|
locked = true;
|
|
|
user_mem = kmap_atomic(page);
|
|
|
|
|
@@ -454,28 +466,25 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
|
|
|
zram_set_flag(meta, index, ZRAM_ZERO);
|
|
|
write_unlock(&zram->meta->tb_lock);
|
|
|
|
|
|
- atomic_inc(&zram->stats.pages_zero);
|
|
|
+ atomic64_inc(&zram->stats.zero_pages);
|
|
|
ret = 0;
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
- ret = lzo1x_1_compress(uncmem, PAGE_SIZE, src, &clen,
|
|
|
- meta->compress_workmem);
|
|
|
+ ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
|
|
|
if (!is_partial_io(bvec)) {
|
|
|
kunmap_atomic(user_mem);
|
|
|
user_mem = NULL;
|
|
|
uncmem = NULL;
|
|
|
}
|
|
|
|
|
|
- if (unlikely(ret != LZO_E_OK)) {
|
|
|
+ if (unlikely(ret)) {
|
|
|
pr_err("Compression failed! err=%d\n", ret);
|
|
|
goto out;
|
|
|
}
|
|
|
-
|
|
|
+ src = zstrm->buffer;
|
|
|
if (unlikely(clen > max_zpage_size)) {
|
|
|
- atomic_inc(&zram->stats.bad_compress);
|
|
|
clen = PAGE_SIZE;
|
|
|
- src = NULL;
|
|
|
if (is_partial_io(bvec))
|
|
|
src = uncmem;
|
|
|
}
|
|
@@ -497,6 +506,8 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
|
|
|
memcpy(cmem, src, clen);
|
|
|
}
|
|
|
|
|
|
+ zcomp_strm_release(zram->comp, zstrm);
|
|
|
+ locked = false;
|
|
|
zs_unmap_object(meta->mem_pool, handle);
|
|
|
|
|
|
/*
|
|
@@ -511,49 +522,88 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
|
|
|
write_unlock(&zram->meta->tb_lock);
|
|
|
|
|
|
/* Update stats */
|
|
|
- atomic64_add(clen, &zram->stats.compr_size);
|
|
|
- atomic_inc(&zram->stats.pages_stored);
|
|
|
- if (clen <= PAGE_SIZE / 2)
|
|
|
- atomic_inc(&zram->stats.good_compress);
|
|
|
-
|
|
|
+ atomic64_add(clen, &zram->stats.compr_data_size);
|
|
|
+ atomic64_inc(&zram->stats.pages_stored);
|
|
|
out:
|
|
|
if (locked)
|
|
|
- mutex_unlock(&meta->buffer_lock);
|
|
|
+ zcomp_strm_release(zram->comp, zstrm);
|
|
|
if (is_partial_io(bvec))
|
|
|
kfree(uncmem);
|
|
|
-
|
|
|
if (ret)
|
|
|
atomic64_inc(&zram->stats.failed_writes);
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
|
|
|
- int offset, struct bio *bio, int rw)
|
|
|
+ int offset, struct bio *bio)
|
|
|
{
|
|
|
int ret;
|
|
|
+ int rw = bio_data_dir(bio);
|
|
|
|
|
|
- if (rw == READ)
|
|
|
+ if (rw == READ) {
|
|
|
+ atomic64_inc(&zram->stats.num_reads);
|
|
|
ret = zram_bvec_read(zram, bvec, index, offset, bio);
|
|
|
- else
|
|
|
+ } else {
|
|
|
+ atomic64_inc(&zram->stats.num_writes);
|
|
|
ret = zram_bvec_write(zram, bvec, index, offset);
|
|
|
+ }
|
|
|
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
+/*
|
|
|
+ * zram_bio_discard - handler on discard request
|
|
|
+ * @index: physical block index in PAGE_SIZE units
|
|
|
+ * @offset: byte offset within physical block
|
|
|
+ */
|
|
|
+static void zram_bio_discard(struct zram *zram, u32 index,
|
|
|
+ int offset, struct bio *bio)
|
|
|
+{
|
|
|
+ size_t n = bio->bi_iter.bi_size;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * zram manages data in physical block size units. Because logical block
|
|
|
+ * size isn't identical with physical block size on some arch, we
|
|
|
+ * could get a discard request pointing to a specific offset within a
|
|
|
+ * certain physical block. Although we can handle this request by
|
|
|
+ * reading that physiclal block and decompressing and partially zeroing
|
|
|
+ * and re-compressing and then re-storing it, this isn't reasonable
|
|
|
+ * because our intent with a discard request is to save memory. So
|
|
|
+ * skipping this logical block is appropriate here.
|
|
|
+ */
|
|
|
+ if (offset) {
|
|
|
+ if (n < offset)
|
|
|
+ return;
|
|
|
+
|
|
|
+ n -= offset;
|
|
|
+ index++;
|
|
|
+ }
|
|
|
+
|
|
|
+ while (n >= PAGE_SIZE) {
|
|
|
+ /*
|
|
|
+ * Discard request can be large so the lock hold times could be
|
|
|
+ * lengthy. So take the lock once per page.
|
|
|
+ */
|
|
|
+ write_lock(&zram->meta->tb_lock);
|
|
|
+ zram_free_page(zram, index);
|
|
|
+ write_unlock(&zram->meta->tb_lock);
|
|
|
+ index++;
|
|
|
+ n -= PAGE_SIZE;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
static void zram_reset_device(struct zram *zram, bool reset_capacity)
|
|
|
{
|
|
|
size_t index;
|
|
|
struct zram_meta *meta;
|
|
|
|
|
|
down_write(&zram->init_lock);
|
|
|
- if (!zram->init_done) {
|
|
|
+ if (!init_done(zram)) {
|
|
|
up_write(&zram->init_lock);
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
meta = zram->meta;
|
|
|
- zram->init_done = 0;
|
|
|
-
|
|
|
/* Free all pages that are still in this zram device */
|
|
|
for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
|
|
|
unsigned long handle = meta->table[index].handle;
|
|
@@ -563,6 +613,9 @@ static void zram_reset_device(struct zram *zram, bool reset_capacity)
|
|
|
zs_free(meta->mem_pool, handle);
|
|
|
}
|
|
|
|
|
|
+ zcomp_destroy(zram->comp);
|
|
|
+ zram->max_comp_streams = 1;
|
|
|
+
|
|
|
zram_meta_free(zram->meta);
|
|
|
zram->meta = NULL;
|
|
|
/* Reset stats */
|
|
@@ -574,37 +627,14 @@ static void zram_reset_device(struct zram *zram, bool reset_capacity)
|
|
|
up_write(&zram->init_lock);
|
|
|
}
|
|
|
|
|
|
-static void zram_init_device(struct zram *zram, struct zram_meta *meta)
|
|
|
-{
|
|
|
- if (zram->disksize > 2 * (totalram_pages << PAGE_SHIFT)) {
|
|
|
- pr_info(
|
|
|
- "There is little point creating a zram of greater than "
|
|
|
- "twice the size of memory since we expect a 2:1 compression "
|
|
|
- "ratio. Note that zram uses about 0.1%% of the size of "
|
|
|
- "the disk when not in use so a huge zram is "
|
|
|
- "wasteful.\n"
|
|
|
- "\tMemory Size: %lu kB\n"
|
|
|
- "\tSize you selected: %llu kB\n"
|
|
|
- "Continuing anyway ...\n",
|
|
|
- (totalram_pages << PAGE_SHIFT) >> 10, zram->disksize >> 10
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- /* zram devices sort of resembles non-rotational disks */
|
|
|
- queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
|
|
|
-
|
|
|
- zram->meta = meta;
|
|
|
- zram->init_done = 1;
|
|
|
-
|
|
|
- pr_debug("Initialization done!\n");
|
|
|
-}
|
|
|
-
|
|
|
static ssize_t disksize_store(struct device *dev,
|
|
|
struct device_attribute *attr, const char *buf, size_t len)
|
|
|
{
|
|
|
u64 disksize;
|
|
|
+ struct zcomp *comp;
|
|
|
struct zram_meta *meta;
|
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
+ int err;
|
|
|
|
|
|
disksize = memparse(buf, NULL);
|
|
|
if (!disksize)
|
|
@@ -614,20 +644,35 @@ static ssize_t disksize_store(struct device *dev,
|
|
|
meta = zram_meta_alloc(disksize);
|
|
|
if (!meta)
|
|
|
return -ENOMEM;
|
|
|
+
|
|
|
+ comp = zcomp_create(zram->compressor, zram->max_comp_streams);
|
|
|
+ if (IS_ERR(comp)) {
|
|
|
+ pr_info("Cannot initialise %s compressing backend\n",
|
|
|
+ zram->compressor);
|
|
|
+ err = PTR_ERR(comp);
|
|
|
+ goto out_free_meta;
|
|
|
+ }
|
|
|
+
|
|
|
down_write(&zram->init_lock);
|
|
|
- if (zram->init_done) {
|
|
|
- up_write(&zram->init_lock);
|
|
|
- zram_meta_free(meta);
|
|
|
+ if (init_done(zram)) {
|
|
|
pr_info("Cannot change disksize for initialized device\n");
|
|
|
- return -EBUSY;
|
|
|
+ err = -EBUSY;
|
|
|
+ goto out_destroy_comp;
|
|
|
}
|
|
|
|
|
|
+ zram->meta = meta;
|
|
|
+ zram->comp = comp;
|
|
|
zram->disksize = disksize;
|
|
|
set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
|
|
|
- zram_init_device(zram, meta);
|
|
|
up_write(&zram->init_lock);
|
|
|
-
|
|
|
return len;
|
|
|
+
|
|
|
+out_destroy_comp:
|
|
|
+ up_write(&zram->init_lock);
|
|
|
+ zcomp_destroy(comp);
|
|
|
+out_free_meta:
|
|
|
+ zram_meta_free(meta);
|
|
|
+ return err;
|
|
|
}
|
|
|
|
|
|
static ssize_t reset_store(struct device *dev,
|
|
@@ -671,26 +716,23 @@ out:
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
-static void __zram_make_request(struct zram *zram, struct bio *bio, int rw)
|
|
|
+static void __zram_make_request(struct zram *zram, struct bio *bio)
|
|
|
{
|
|
|
int offset;
|
|
|
u32 index;
|
|
|
struct bio_vec bvec;
|
|
|
struct bvec_iter iter;
|
|
|
|
|
|
- switch (rw) {
|
|
|
- case READ:
|
|
|
- atomic64_inc(&zram->stats.num_reads);
|
|
|
- break;
|
|
|
- case WRITE:
|
|
|
- atomic64_inc(&zram->stats.num_writes);
|
|
|
- break;
|
|
|
- }
|
|
|
-
|
|
|
index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
|
|
|
offset = (bio->bi_iter.bi_sector &
|
|
|
(SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
|
|
|
|
|
|
+ if (unlikely(bio->bi_rw & REQ_DISCARD)) {
|
|
|
+ zram_bio_discard(zram, index, offset, bio);
|
|
|
+ bio_endio(bio, 0);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
bio_for_each_segment(bvec, bio, iter) {
|
|
|
int max_transfer_size = PAGE_SIZE - offset;
|
|
|
|
|
@@ -705,16 +747,15 @@ static void __zram_make_request(struct zram *zram, struct bio *bio, int rw)
|
|
|
bv.bv_len = max_transfer_size;
|
|
|
bv.bv_offset = bvec.bv_offset;
|
|
|
|
|
|
- if (zram_bvec_rw(zram, &bv, index, offset, bio, rw) < 0)
|
|
|
+ if (zram_bvec_rw(zram, &bv, index, offset, bio) < 0)
|
|
|
goto out;
|
|
|
|
|
|
bv.bv_len = bvec.bv_len - max_transfer_size;
|
|
|
bv.bv_offset += max_transfer_size;
|
|
|
- if (zram_bvec_rw(zram, &bv, index+1, 0, bio, rw) < 0)
|
|
|
+ if (zram_bvec_rw(zram, &bv, index + 1, 0, bio) < 0)
|
|
|
goto out;
|
|
|
} else
|
|
|
- if (zram_bvec_rw(zram, &bvec, index, offset, bio, rw)
|
|
|
- < 0)
|
|
|
+ if (zram_bvec_rw(zram, &bvec, index, offset, bio) < 0)
|
|
|
goto out;
|
|
|
|
|
|
update_position(&index, &offset, &bvec);
|
|
@@ -736,7 +777,7 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio)
|
|
|
struct zram *zram = queue->queuedata;
|
|
|
|
|
|
down_read(&zram->init_lock);
|
|
|
- if (unlikely(!zram->init_done))
|
|
|
+ if (unlikely(!init_done(zram)))
|
|
|
goto error;
|
|
|
|
|
|
if (!valid_io_request(zram, bio)) {
|
|
@@ -744,7 +785,7 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio)
|
|
|
goto error;
|
|
|
}
|
|
|
|
|
|
- __zram_make_request(zram, bio, bio_data_dir(bio));
|
|
|
+ __zram_make_request(zram, bio);
|
|
|
up_read(&zram->init_lock);
|
|
|
|
|
|
return;
|
|
@@ -778,14 +819,21 @@ static DEVICE_ATTR(disksize, S_IRUGO | S_IWUSR,
|
|
|
disksize_show, disksize_store);
|
|
|
static DEVICE_ATTR(initstate, S_IRUGO, initstate_show, NULL);
|
|
|
static DEVICE_ATTR(reset, S_IWUSR, NULL, reset_store);
|
|
|
-static DEVICE_ATTR(num_reads, S_IRUGO, num_reads_show, NULL);
|
|
|
-static DEVICE_ATTR(num_writes, S_IRUGO, num_writes_show, NULL);
|
|
|
-static DEVICE_ATTR(invalid_io, S_IRUGO, invalid_io_show, NULL);
|
|
|
-static DEVICE_ATTR(notify_free, S_IRUGO, notify_free_show, NULL);
|
|
|
-static DEVICE_ATTR(zero_pages, S_IRUGO, zero_pages_show, NULL);
|
|
|
static DEVICE_ATTR(orig_data_size, S_IRUGO, orig_data_size_show, NULL);
|
|
|
-static DEVICE_ATTR(compr_data_size, S_IRUGO, compr_data_size_show, NULL);
|
|
|
static DEVICE_ATTR(mem_used_total, S_IRUGO, mem_used_total_show, NULL);
|
|
|
+static DEVICE_ATTR(max_comp_streams, S_IRUGO | S_IWUSR,
|
|
|
+ max_comp_streams_show, max_comp_streams_store);
|
|
|
+static DEVICE_ATTR(comp_algorithm, S_IRUGO | S_IWUSR,
|
|
|
+ comp_algorithm_show, comp_algorithm_store);
|
|
|
+
|
|
|
+ZRAM_ATTR_RO(num_reads);
|
|
|
+ZRAM_ATTR_RO(num_writes);
|
|
|
+ZRAM_ATTR_RO(failed_reads);
|
|
|
+ZRAM_ATTR_RO(failed_writes);
|
|
|
+ZRAM_ATTR_RO(invalid_io);
|
|
|
+ZRAM_ATTR_RO(notify_free);
|
|
|
+ZRAM_ATTR_RO(zero_pages);
|
|
|
+ZRAM_ATTR_RO(compr_data_size);
|
|
|
|
|
|
static struct attribute *zram_disk_attrs[] = {
|
|
|
&dev_attr_disksize.attr,
|
|
@@ -793,12 +841,16 @@ static struct attribute *zram_disk_attrs[] = {
|
|
|
&dev_attr_reset.attr,
|
|
|
&dev_attr_num_reads.attr,
|
|
|
&dev_attr_num_writes.attr,
|
|
|
+ &dev_attr_failed_reads.attr,
|
|
|
+ &dev_attr_failed_writes.attr,
|
|
|
&dev_attr_invalid_io.attr,
|
|
|
&dev_attr_notify_free.attr,
|
|
|
&dev_attr_zero_pages.attr,
|
|
|
&dev_attr_orig_data_size.attr,
|
|
|
&dev_attr_compr_data_size.attr,
|
|
|
&dev_attr_mem_used_total.attr,
|
|
|
+ &dev_attr_max_comp_streams.attr,
|
|
|
+ &dev_attr_comp_algorithm.attr,
|
|
|
NULL,
|
|
|
};
|
|
|
|
|
@@ -839,7 +891,8 @@ static int create_device(struct zram *zram, int device_id)
|
|
|
|
|
|
/* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
|
|
|
set_capacity(zram->disk, 0);
|
|
|
-
|
|
|
+ /* zram devices sort of resembles non-rotational disks */
|
|
|
+ queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
|
|
|
/*
|
|
|
* To ensure that we always get PAGE_SIZE aligned
|
|
|
* and n*PAGE_SIZED sized I/O requests.
|
|
@@ -849,6 +902,21 @@ static int create_device(struct zram *zram, int device_id)
|
|
|
ZRAM_LOGICAL_BLOCK_SIZE);
|
|
|
blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
|
|
|
blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
|
|
|
+ zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
|
|
|
+ zram->disk->queue->limits.max_discard_sectors = UINT_MAX;
|
|
|
+ /*
|
|
|
+ * zram_bio_discard() will clear all logical blocks if logical block
|
|
|
+ * size is identical with physical block size(PAGE_SIZE). But if it is
|
|
|
+ * different, we will skip discarding some parts of logical blocks in
|
|
|
+ * the part of the request range which isn't aligned to physical block
|
|
|
+ * size. So we can't ensure that all discarded logical blocks are
|
|
|
+ * zeroed.
|
|
|
+ */
|
|
|
+ if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
|
|
|
+ zram->disk->queue->limits.discard_zeroes_data = 1;
|
|
|
+ else
|
|
|
+ zram->disk->queue->limits.discard_zeroes_data = 0;
|
|
|
+ queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);
|
|
|
|
|
|
add_disk(zram->disk);
|
|
|
|
|
@@ -858,8 +926,9 @@ static int create_device(struct zram *zram, int device_id)
|
|
|
pr_warn("Error creating sysfs group");
|
|
|
goto out_free_disk;
|
|
|
}
|
|
|
-
|
|
|
- zram->init_done = 0;
|
|
|
+ strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
|
|
|
+ zram->meta = NULL;
|
|
|
+ zram->max_comp_streams = 1;
|
|
|
return 0;
|
|
|
|
|
|
out_free_disk:
|