|
@@ -14,12 +14,11 @@
|
|
* access behavior.
|
|
* access behavior.
|
|
*/
|
|
*/
|
|
|
|
|
|
-#include "dm-bufio.h"
|
|
|
|
|
|
+#include "dm-verity.h"
|
|
|
|
+#include "dm-verity-fec.h"
|
|
|
|
|
|
#include <linux/module.h>
|
|
#include <linux/module.h>
|
|
-#include <linux/device-mapper.h>
|
|
|
|
#include <linux/reboot.h>
|
|
#include <linux/reboot.h>
|
|
-#include <crypto/hash.h>
|
|
|
|
|
|
|
|
#define DM_MSG_PREFIX "verity"
|
|
#define DM_MSG_PREFIX "verity"
|
|
|
|
|
|
@@ -28,83 +27,18 @@
|
|
|
|
|
|
#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
|
|
#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
|
|
|
|
|
|
-#define DM_VERITY_MAX_LEVELS 63
|
|
|
|
#define DM_VERITY_MAX_CORRUPTED_ERRS 100
|
|
#define DM_VERITY_MAX_CORRUPTED_ERRS 100
|
|
|
|
|
|
#define DM_VERITY_OPT_LOGGING "ignore_corruption"
|
|
#define DM_VERITY_OPT_LOGGING "ignore_corruption"
|
|
#define DM_VERITY_OPT_RESTART "restart_on_corruption"
|
|
#define DM_VERITY_OPT_RESTART "restart_on_corruption"
|
|
|
|
+#define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks"
|
|
|
|
+
|
|
|
|
+#define DM_VERITY_OPTS_MAX (2 + DM_VERITY_OPTS_FEC)
|
|
|
|
|
|
static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
|
|
static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
|
|
|
|
|
|
module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR);
|
|
module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR);
|
|
|
|
|
|
-enum verity_mode {
|
|
|
|
- DM_VERITY_MODE_EIO,
|
|
|
|
- DM_VERITY_MODE_LOGGING,
|
|
|
|
- DM_VERITY_MODE_RESTART
|
|
|
|
-};
|
|
|
|
-
|
|
|
|
-enum verity_block_type {
|
|
|
|
- DM_VERITY_BLOCK_TYPE_DATA,
|
|
|
|
- DM_VERITY_BLOCK_TYPE_METADATA
|
|
|
|
-};
|
|
|
|
-
|
|
|
|
-struct dm_verity {
|
|
|
|
- struct dm_dev *data_dev;
|
|
|
|
- struct dm_dev *hash_dev;
|
|
|
|
- struct dm_target *ti;
|
|
|
|
- struct dm_bufio_client *bufio;
|
|
|
|
- char *alg_name;
|
|
|
|
- struct crypto_shash *tfm;
|
|
|
|
- u8 *root_digest; /* digest of the root block */
|
|
|
|
- u8 *salt; /* salt: its size is salt_size */
|
|
|
|
- unsigned salt_size;
|
|
|
|
- sector_t data_start; /* data offset in 512-byte sectors */
|
|
|
|
- sector_t hash_start; /* hash start in blocks */
|
|
|
|
- sector_t data_blocks; /* the number of data blocks */
|
|
|
|
- sector_t hash_blocks; /* the number of hash blocks */
|
|
|
|
- unsigned char data_dev_block_bits; /* log2(data blocksize) */
|
|
|
|
- unsigned char hash_dev_block_bits; /* log2(hash blocksize) */
|
|
|
|
- unsigned char hash_per_block_bits; /* log2(hashes in hash block) */
|
|
|
|
- unsigned char levels; /* the number of tree levels */
|
|
|
|
- unsigned char version;
|
|
|
|
- unsigned digest_size; /* digest size for the current hash algorithm */
|
|
|
|
- unsigned shash_descsize;/* the size of temporary space for crypto */
|
|
|
|
- int hash_failed; /* set to 1 if hash of any block failed */
|
|
|
|
- enum verity_mode mode; /* mode for handling verification errors */
|
|
|
|
- unsigned corrupted_errs;/* Number of errors for corrupted blocks */
|
|
|
|
-
|
|
|
|
- struct workqueue_struct *verify_wq;
|
|
|
|
-
|
|
|
|
- /* starting blocks for each tree level. 0 is the lowest level. */
|
|
|
|
- sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
|
|
|
|
-};
|
|
|
|
-
|
|
|
|
-struct dm_verity_io {
|
|
|
|
- struct dm_verity *v;
|
|
|
|
-
|
|
|
|
- /* original values of bio->bi_end_io and bio->bi_private */
|
|
|
|
- bio_end_io_t *orig_bi_end_io;
|
|
|
|
- void *orig_bi_private;
|
|
|
|
-
|
|
|
|
- sector_t block;
|
|
|
|
- unsigned n_blocks;
|
|
|
|
-
|
|
|
|
- struct bvec_iter iter;
|
|
|
|
-
|
|
|
|
- struct work_struct work;
|
|
|
|
-
|
|
|
|
- /*
|
|
|
|
- * Three variably-size fields follow this struct:
|
|
|
|
- *
|
|
|
|
- * u8 hash_desc[v->shash_descsize];
|
|
|
|
- * u8 real_digest[v->digest_size];
|
|
|
|
- * u8 want_digest[v->digest_size];
|
|
|
|
- *
|
|
|
|
- * To access them use: io_hash_desc(), io_real_digest() and io_want_digest().
|
|
|
|
- */
|
|
|
|
-};
|
|
|
|
-
|
|
|
|
struct dm_verity_prefetch_work {
|
|
struct dm_verity_prefetch_work {
|
|
struct work_struct work;
|
|
struct work_struct work;
|
|
struct dm_verity *v;
|
|
struct dm_verity *v;
|
|
@@ -112,21 +46,6 @@ struct dm_verity_prefetch_work {
|
|
unsigned n_blocks;
|
|
unsigned n_blocks;
|
|
};
|
|
};
|
|
|
|
|
|
-static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
|
|
|
|
-{
|
|
|
|
- return (struct shash_desc *)(io + 1);
|
|
|
|
-}
|
|
|
|
-
|
|
|
|
-static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
|
|
|
|
-{
|
|
|
|
- return (u8 *)(io + 1) + v->shash_descsize;
|
|
|
|
-}
|
|
|
|
-
|
|
|
|
-static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
|
|
|
|
-{
|
|
|
|
- return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
|
|
|
|
-}
|
|
|
|
-
|
|
|
|
/*
|
|
/*
|
|
* Auxiliary structure appended to each dm-bufio buffer. If the value
|
|
* Auxiliary structure appended to each dm-bufio buffer. If the value
|
|
* hash_verified is nonzero, hash of the block has been verified.
|
|
* hash_verified is nonzero, hash of the block has been verified.
|
|
@@ -173,6 +92,84 @@ static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
|
|
return block >> (level * v->hash_per_block_bits);
|
|
return block >> (level * v->hash_per_block_bits);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+/*
|
|
|
|
+ * Wrapper for crypto_shash_init, which handles verity salting.
|
|
|
|
+ */
|
|
|
|
+static int verity_hash_init(struct dm_verity *v, struct shash_desc *desc)
|
|
|
|
+{
|
|
|
|
+ int r;
|
|
|
|
+
|
|
|
|
+ desc->tfm = v->tfm;
|
|
|
|
+ desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
|
|
|
|
+
|
|
|
|
+ r = crypto_shash_init(desc);
|
|
|
|
+
|
|
|
|
+ if (unlikely(r < 0)) {
|
|
|
|
+ DMERR("crypto_shash_init failed: %d", r);
|
|
|
|
+ return r;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if (likely(v->version >= 1)) {
|
|
|
|
+ r = crypto_shash_update(desc, v->salt, v->salt_size);
|
|
|
|
+
|
|
|
|
+ if (unlikely(r < 0)) {
|
|
|
|
+ DMERR("crypto_shash_update failed: %d", r);
|
|
|
|
+ return r;
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ return 0;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static int verity_hash_update(struct dm_verity *v, struct shash_desc *desc,
|
|
|
|
+ const u8 *data, size_t len)
|
|
|
|
+{
|
|
|
|
+ int r = crypto_shash_update(desc, data, len);
|
|
|
|
+
|
|
|
|
+ if (unlikely(r < 0))
|
|
|
|
+ DMERR("crypto_shash_update failed: %d", r);
|
|
|
|
+
|
|
|
|
+ return r;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static int verity_hash_final(struct dm_verity *v, struct shash_desc *desc,
|
|
|
|
+ u8 *digest)
|
|
|
|
+{
|
|
|
|
+ int r;
|
|
|
|
+
|
|
|
|
+ if (unlikely(!v->version)) {
|
|
|
|
+ r = crypto_shash_update(desc, v->salt, v->salt_size);
|
|
|
|
+
|
|
|
|
+ if (r < 0) {
|
|
|
|
+ DMERR("crypto_shash_update failed: %d", r);
|
|
|
|
+ return r;
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ r = crypto_shash_final(desc, digest);
|
|
|
|
+
|
|
|
|
+ if (unlikely(r < 0))
|
|
|
|
+ DMERR("crypto_shash_final failed: %d", r);
|
|
|
|
+
|
|
|
|
+ return r;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+int verity_hash(struct dm_verity *v, struct shash_desc *desc,
|
|
|
|
+ const u8 *data, size_t len, u8 *digest)
|
|
|
|
+{
|
|
|
|
+ int r;
|
|
|
|
+
|
|
|
|
+ r = verity_hash_init(v, desc);
|
|
|
|
+ if (unlikely(r < 0))
|
|
|
|
+ return r;
|
|
|
|
+
|
|
|
|
+ r = verity_hash_update(v, desc, data, len);
|
|
|
|
+ if (unlikely(r < 0))
|
|
|
|
+ return r;
|
|
|
|
+
|
|
|
|
+ return verity_hash_final(v, desc, digest);
|
|
|
|
+}
|
|
|
|
+
|
|
static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
|
|
static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
|
|
sector_t *hash_block, unsigned *offset)
|
|
sector_t *hash_block, unsigned *offset)
|
|
{
|
|
{
|
|
@@ -246,17 +243,17 @@ out:
|
|
* Verify hash of a metadata block pertaining to the specified data block
|
|
* Verify hash of a metadata block pertaining to the specified data block
|
|
* ("block" argument) at a specified level ("level" argument).
|
|
* ("block" argument) at a specified level ("level" argument).
|
|
*
|
|
*
|
|
- * On successful return, io_want_digest(v, io) contains the hash value for
|
|
|
|
- * a lower tree level or for the data block (if we're at the lowest leve).
|
|
|
|
|
|
+ * On successful return, verity_io_want_digest(v, io) contains the hash value
|
|
|
|
+ * for a lower tree level or for the data block (if we're at the lowest level).
|
|
*
|
|
*
|
|
* If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
|
|
* If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
|
|
* If "skip_unverified" is false, unverified buffer is hashed and verified
|
|
* If "skip_unverified" is false, unverified buffer is hashed and verified
|
|
- * against current value of io_want_digest(v, io).
|
|
|
|
|
|
+ * against current value of verity_io_want_digest(v, io).
|
|
*/
|
|
*/
|
|
-static int verity_verify_level(struct dm_verity_io *io, sector_t block,
|
|
|
|
- int level, bool skip_unverified)
|
|
|
|
|
|
+static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
|
|
|
|
+ sector_t block, int level, bool skip_unverified,
|
|
|
|
+ u8 *want_digest)
|
|
{
|
|
{
|
|
- struct dm_verity *v = io->v;
|
|
|
|
struct dm_buffer *buf;
|
|
struct dm_buffer *buf;
|
|
struct buffer_aux *aux;
|
|
struct buffer_aux *aux;
|
|
u8 *data;
|
|
u8 *data;
|
|
@@ -273,72 +270,128 @@ static int verity_verify_level(struct dm_verity_io *io, sector_t block,
|
|
aux = dm_bufio_get_aux_data(buf);
|
|
aux = dm_bufio_get_aux_data(buf);
|
|
|
|
|
|
if (!aux->hash_verified) {
|
|
if (!aux->hash_verified) {
|
|
- struct shash_desc *desc;
|
|
|
|
- u8 *result;
|
|
|
|
-
|
|
|
|
if (skip_unverified) {
|
|
if (skip_unverified) {
|
|
r = 1;
|
|
r = 1;
|
|
goto release_ret_r;
|
|
goto release_ret_r;
|
|
}
|
|
}
|
|
|
|
|
|
- desc = io_hash_desc(v, io);
|
|
|
|
- desc->tfm = v->tfm;
|
|
|
|
- desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
|
|
|
|
- r = crypto_shash_init(desc);
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_init failed: %d", r);
|
|
|
|
|
|
+ r = verity_hash(v, verity_io_hash_desc(v, io),
|
|
|
|
+ data, 1 << v->hash_dev_block_bits,
|
|
|
|
+ verity_io_real_digest(v, io));
|
|
|
|
+ if (unlikely(r < 0))
|
|
goto release_ret_r;
|
|
goto release_ret_r;
|
|
- }
|
|
|
|
-
|
|
|
|
- if (likely(v->version >= 1)) {
|
|
|
|
- r = crypto_shash_update(desc, v->salt, v->salt_size);
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_update failed: %d", r);
|
|
|
|
- goto release_ret_r;
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
|
|
|
|
- r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits);
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_update failed: %d", r);
|
|
|
|
|
|
+ if (likely(memcmp(verity_io_real_digest(v, io), want_digest,
|
|
|
|
+ v->digest_size) == 0))
|
|
|
|
+ aux->hash_verified = 1;
|
|
|
|
+ else if (verity_fec_decode(v, io,
|
|
|
|
+ DM_VERITY_BLOCK_TYPE_METADATA,
|
|
|
|
+ hash_block, data, NULL) == 0)
|
|
|
|
+ aux->hash_verified = 1;
|
|
|
|
+ else if (verity_handle_err(v,
|
|
|
|
+ DM_VERITY_BLOCK_TYPE_METADATA,
|
|
|
|
+ hash_block)) {
|
|
|
|
+ r = -EIO;
|
|
goto release_ret_r;
|
|
goto release_ret_r;
|
|
}
|
|
}
|
|
|
|
+ }
|
|
|
|
|
|
- if (!v->version) {
|
|
|
|
- r = crypto_shash_update(desc, v->salt, v->salt_size);
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_update failed: %d", r);
|
|
|
|
- goto release_ret_r;
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
|
|
+ data += offset;
|
|
|
|
+ memcpy(want_digest, data, v->digest_size);
|
|
|
|
+ r = 0;
|
|
|
|
|
|
- result = io_real_digest(v, io);
|
|
|
|
- r = crypto_shash_final(desc, result);
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_final failed: %d", r);
|
|
|
|
- goto release_ret_r;
|
|
|
|
- }
|
|
|
|
- if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
|
|
|
|
- if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_METADATA,
|
|
|
|
- hash_block)) {
|
|
|
|
- r = -EIO;
|
|
|
|
- goto release_ret_r;
|
|
|
|
- }
|
|
|
|
- } else
|
|
|
|
- aux->hash_verified = 1;
|
|
|
|
|
|
+release_ret_r:
|
|
|
|
+ dm_bufio_release(buf);
|
|
|
|
+ return r;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/*
|
|
|
|
+ * Find a hash for a given block, write it to digest and verify the integrity
|
|
|
|
+ * of the hash tree if necessary.
|
|
|
|
+ */
|
|
|
|
+int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
|
|
|
|
+ sector_t block, u8 *digest, bool *is_zero)
|
|
|
|
+{
|
|
|
|
+ int r = 0, i;
|
|
|
|
+
|
|
|
|
+ if (likely(v->levels)) {
|
|
|
|
+ /*
|
|
|
|
+ * First, we try to get the requested hash for
|
|
|
|
+ * the current block. If the hash block itself is
|
|
|
|
+ * verified, zero is returned. If it isn't, this
|
|
|
|
+ * function returns 1 and we fall back to whole
|
|
|
|
+ * chain verification.
|
|
|
|
+ */
|
|
|
|
+ r = verity_verify_level(v, io, block, 0, true, digest);
|
|
|
|
+ if (likely(r <= 0))
|
|
|
|
+ goto out;
|
|
}
|
|
}
|
|
|
|
|
|
- data += offset;
|
|
|
|
|
|
+ memcpy(digest, v->root_digest, v->digest_size);
|
|
|
|
|
|
- memcpy(io_want_digest(v, io), data, v->digest_size);
|
|
|
|
|
|
+ for (i = v->levels - 1; i >= 0; i--) {
|
|
|
|
+ r = verity_verify_level(v, io, block, i, false, digest);
|
|
|
|
+ if (unlikely(r))
|
|
|
|
+ goto out;
|
|
|
|
+ }
|
|
|
|
+out:
|
|
|
|
+ if (!r && v->zero_digest)
|
|
|
|
+ *is_zero = !memcmp(v->zero_digest, digest, v->digest_size);
|
|
|
|
+ else
|
|
|
|
+ *is_zero = false;
|
|
|
|
+
|
|
|
|
+ return r;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/*
|
|
|
|
+ * Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec
|
|
|
|
+ * starting from iter.
|
|
|
|
+ */
|
|
|
|
+int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io,
|
|
|
|
+ struct bvec_iter *iter,
|
|
|
|
+ int (*process)(struct dm_verity *v,
|
|
|
|
+ struct dm_verity_io *io, u8 *data,
|
|
|
|
+ size_t len))
|
|
|
|
+{
|
|
|
|
+ unsigned todo = 1 << v->data_dev_block_bits;
|
|
|
|
+ struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size);
|
|
|
|
+
|
|
|
|
+ do {
|
|
|
|
+ int r;
|
|
|
|
+ u8 *page;
|
|
|
|
+ unsigned len;
|
|
|
|
+ struct bio_vec bv = bio_iter_iovec(bio, *iter);
|
|
|
|
+
|
|
|
|
+ page = kmap_atomic(bv.bv_page);
|
|
|
|
+ len = bv.bv_len;
|
|
|
|
+
|
|
|
|
+ if (likely(len >= todo))
|
|
|
|
+ len = todo;
|
|
|
|
+
|
|
|
|
+ r = process(v, io, page + bv.bv_offset, len);
|
|
|
|
+ kunmap_atomic(page);
|
|
|
|
+
|
|
|
|
+ if (r < 0)
|
|
|
|
+ return r;
|
|
|
|
+
|
|
|
|
+ bio_advance_iter(bio, iter, len);
|
|
|
|
+ todo -= len;
|
|
|
|
+ } while (todo);
|
|
|
|
|
|
- dm_bufio_release(buf);
|
|
|
|
return 0;
|
|
return 0;
|
|
|
|
+}
|
|
|
|
|
|
-release_ret_r:
|
|
|
|
- dm_bufio_release(buf);
|
|
|
|
|
|
+static int verity_bv_hash_update(struct dm_verity *v, struct dm_verity_io *io,
|
|
|
|
+ u8 *data, size_t len)
|
|
|
|
+{
|
|
|
|
+ return verity_hash_update(v, verity_io_hash_desc(v, io), data, len);
|
|
|
|
+}
|
|
|
|
|
|
- return r;
|
|
|
|
|
|
+static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io,
|
|
|
|
+ u8 *data, size_t len)
|
|
|
|
+{
|
|
|
|
+ memset(data, 0, len);
|
|
|
|
+ return 0;
|
|
}
|
|
}
|
|
|
|
|
|
/*
|
|
/*
|
|
@@ -346,99 +399,56 @@ release_ret_r:
|
|
*/
|
|
*/
|
|
static int verity_verify_io(struct dm_verity_io *io)
|
|
static int verity_verify_io(struct dm_verity_io *io)
|
|
{
|
|
{
|
|
|
|
+ bool is_zero;
|
|
struct dm_verity *v = io->v;
|
|
struct dm_verity *v = io->v;
|
|
- struct bio *bio = dm_bio_from_per_bio_data(io,
|
|
|
|
- v->ti->per_bio_data_size);
|
|
|
|
|
|
+ struct bvec_iter start;
|
|
unsigned b;
|
|
unsigned b;
|
|
- int i;
|
|
|
|
|
|
|
|
for (b = 0; b < io->n_blocks; b++) {
|
|
for (b = 0; b < io->n_blocks; b++) {
|
|
- struct shash_desc *desc;
|
|
|
|
- u8 *result;
|
|
|
|
int r;
|
|
int r;
|
|
- unsigned todo;
|
|
|
|
|
|
+ struct shash_desc *desc = verity_io_hash_desc(v, io);
|
|
|
|
+
|
|
|
|
+ r = verity_hash_for_block(v, io, io->block + b,
|
|
|
|
+ verity_io_want_digest(v, io),
|
|
|
|
+ &is_zero);
|
|
|
|
+ if (unlikely(r < 0))
|
|
|
|
+ return r;
|
|
|
|
|
|
- if (likely(v->levels)) {
|
|
|
|
|
|
+ if (is_zero) {
|
|
/*
|
|
/*
|
|
- * First, we try to get the requested hash for
|
|
|
|
- * the current block. If the hash block itself is
|
|
|
|
- * verified, zero is returned. If it isn't, this
|
|
|
|
- * function returns 0 and we fall back to whole
|
|
|
|
- * chain verification.
|
|
|
|
|
|
+ * If we expect a zero block, don't validate, just
|
|
|
|
+ * return zeros.
|
|
*/
|
|
*/
|
|
- int r = verity_verify_level(io, io->block + b, 0, true);
|
|
|
|
- if (likely(!r))
|
|
|
|
- goto test_block_hash;
|
|
|
|
- if (r < 0)
|
|
|
|
|
|
+ r = verity_for_bv_block(v, io, &io->iter,
|
|
|
|
+ verity_bv_zero);
|
|
|
|
+ if (unlikely(r < 0))
|
|
return r;
|
|
return r;
|
|
- }
|
|
|
|
|
|
|
|
- memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);
|
|
|
|
-
|
|
|
|
- for (i = v->levels - 1; i >= 0; i--) {
|
|
|
|
- int r = verity_verify_level(io, io->block + b, i, false);
|
|
|
|
- if (unlikely(r))
|
|
|
|
- return r;
|
|
|
|
|
|
+ continue;
|
|
}
|
|
}
|
|
|
|
|
|
-test_block_hash:
|
|
|
|
- desc = io_hash_desc(v, io);
|
|
|
|
- desc->tfm = v->tfm;
|
|
|
|
- desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
|
|
|
|
- r = crypto_shash_init(desc);
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_init failed: %d", r);
|
|
|
|
|
|
+ r = verity_hash_init(v, desc);
|
|
|
|
+ if (unlikely(r < 0))
|
|
return r;
|
|
return r;
|
|
- }
|
|
|
|
-
|
|
|
|
- if (likely(v->version >= 1)) {
|
|
|
|
- r = crypto_shash_update(desc, v->salt, v->salt_size);
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_update failed: %d", r);
|
|
|
|
- return r;
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- todo = 1 << v->data_dev_block_bits;
|
|
|
|
- do {
|
|
|
|
- u8 *page;
|
|
|
|
- unsigned len;
|
|
|
|
- struct bio_vec bv = bio_iter_iovec(bio, io->iter);
|
|
|
|
-
|
|
|
|
- page = kmap_atomic(bv.bv_page);
|
|
|
|
- len = bv.bv_len;
|
|
|
|
- if (likely(len >= todo))
|
|
|
|
- len = todo;
|
|
|
|
- r = crypto_shash_update(desc, page + bv.bv_offset, len);
|
|
|
|
- kunmap_atomic(page);
|
|
|
|
-
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_update failed: %d", r);
|
|
|
|
- return r;
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- bio_advance_iter(bio, &io->iter, len);
|
|
|
|
- todo -= len;
|
|
|
|
- } while (todo);
|
|
|
|
|
|
|
|
- if (!v->version) {
|
|
|
|
- r = crypto_shash_update(desc, v->salt, v->salt_size);
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_update failed: %d", r);
|
|
|
|
- return r;
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
|
|
+ start = io->iter;
|
|
|
|
+ r = verity_for_bv_block(v, io, &io->iter, verity_bv_hash_update);
|
|
|
|
+ if (unlikely(r < 0))
|
|
|
|
+ return r;
|
|
|
|
|
|
- result = io_real_digest(v, io);
|
|
|
|
- r = crypto_shash_final(desc, result);
|
|
|
|
- if (r < 0) {
|
|
|
|
- DMERR("crypto_shash_final failed: %d", r);
|
|
|
|
|
|
+ r = verity_hash_final(v, desc, verity_io_real_digest(v, io));
|
|
|
|
+ if (unlikely(r < 0))
|
|
return r;
|
|
return r;
|
|
- }
|
|
|
|
- if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
|
|
|
|
- if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA,
|
|
|
|
- io->block + b))
|
|
|
|
- return -EIO;
|
|
|
|
- }
|
|
|
|
|
|
+
|
|
|
|
+ if (likely(memcmp(verity_io_real_digest(v, io),
|
|
|
|
+ verity_io_want_digest(v, io), v->digest_size) == 0))
|
|
|
|
+ continue;
|
|
|
|
+ else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA,
|
|
|
|
+ io->block + b, NULL, &start) == 0)
|
|
|
|
+ continue;
|
|
|
|
+ else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA,
|
|
|
|
+ io->block + b))
|
|
|
|
+ return -EIO;
|
|
}
|
|
}
|
|
|
|
|
|
return 0;
|
|
return 0;
|
|
@@ -453,9 +463,10 @@ static void verity_finish_io(struct dm_verity_io *io, int error)
|
|
struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size);
|
|
struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size);
|
|
|
|
|
|
bio->bi_end_io = io->orig_bi_end_io;
|
|
bio->bi_end_io = io->orig_bi_end_io;
|
|
- bio->bi_private = io->orig_bi_private;
|
|
|
|
bio->bi_error = error;
|
|
bio->bi_error = error;
|
|
|
|
|
|
|
|
+ verity_fec_finish_io(io);
|
|
|
|
+
|
|
bio_endio(bio);
|
|
bio_endio(bio);
|
|
}
|
|
}
|
|
|
|
|
|
@@ -470,7 +481,7 @@ static void verity_end_io(struct bio *bio)
|
|
{
|
|
{
|
|
struct dm_verity_io *io = bio->bi_private;
|
|
struct dm_verity_io *io = bio->bi_private;
|
|
|
|
|
|
- if (bio->bi_error) {
|
|
|
|
|
|
+ if (bio->bi_error && !verity_fec_is_enabled(io->v)) {
|
|
verity_finish_io(io, bio->bi_error);
|
|
verity_finish_io(io, bio->bi_error);
|
|
return;
|
|
return;
|
|
}
|
|
}
|
|
@@ -566,7 +577,6 @@ static int verity_map(struct dm_target *ti, struct bio *bio)
|
|
io = dm_per_bio_data(bio, ti->per_bio_data_size);
|
|
io = dm_per_bio_data(bio, ti->per_bio_data_size);
|
|
io->v = v;
|
|
io->v = v;
|
|
io->orig_bi_end_io = bio->bi_end_io;
|
|
io->orig_bi_end_io = bio->bi_end_io;
|
|
- io->orig_bi_private = bio->bi_private;
|
|
|
|
io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
|
|
io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
|
|
io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;
|
|
io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;
|
|
|
|
|
|
@@ -574,6 +584,8 @@ static int verity_map(struct dm_target *ti, struct bio *bio)
|
|
bio->bi_private = io;
|
|
bio->bi_private = io;
|
|
io->iter = bio->bi_iter;
|
|
io->iter = bio->bi_iter;
|
|
|
|
|
|
|
|
+ verity_fec_init_io(io);
|
|
|
|
+
|
|
verity_submit_prefetch(v, io);
|
|
verity_submit_prefetch(v, io);
|
|
|
|
|
|
generic_make_request(bio);
|
|
generic_make_request(bio);
|
|
@@ -588,6 +600,7 @@ static void verity_status(struct dm_target *ti, status_type_t type,
|
|
unsigned status_flags, char *result, unsigned maxlen)
|
|
unsigned status_flags, char *result, unsigned maxlen)
|
|
{
|
|
{
|
|
struct dm_verity *v = ti->private;
|
|
struct dm_verity *v = ti->private;
|
|
|
|
+ unsigned args = 0;
|
|
unsigned sz = 0;
|
|
unsigned sz = 0;
|
|
unsigned x;
|
|
unsigned x;
|
|
|
|
|
|
@@ -614,8 +627,17 @@ static void verity_status(struct dm_target *ti, status_type_t type,
|
|
else
|
|
else
|
|
for (x = 0; x < v->salt_size; x++)
|
|
for (x = 0; x < v->salt_size; x++)
|
|
DMEMIT("%02x", v->salt[x]);
|
|
DMEMIT("%02x", v->salt[x]);
|
|
|
|
+ if (v->mode != DM_VERITY_MODE_EIO)
|
|
|
|
+ args++;
|
|
|
|
+ if (verity_fec_is_enabled(v))
|
|
|
|
+ args += DM_VERITY_OPTS_FEC;
|
|
|
|
+ if (v->zero_digest)
|
|
|
|
+ args++;
|
|
|
|
+ if (!args)
|
|
|
|
+ return;
|
|
|
|
+ DMEMIT(" %u", args);
|
|
if (v->mode != DM_VERITY_MODE_EIO) {
|
|
if (v->mode != DM_VERITY_MODE_EIO) {
|
|
- DMEMIT(" 1 ");
|
|
|
|
|
|
+ DMEMIT(" ");
|
|
switch (v->mode) {
|
|
switch (v->mode) {
|
|
case DM_VERITY_MODE_LOGGING:
|
|
case DM_VERITY_MODE_LOGGING:
|
|
DMEMIT(DM_VERITY_OPT_LOGGING);
|
|
DMEMIT(DM_VERITY_OPT_LOGGING);
|
|
@@ -627,6 +649,9 @@ static void verity_status(struct dm_target *ti, status_type_t type,
|
|
BUG();
|
|
BUG();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
+ if (v->zero_digest)
|
|
|
|
+ DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
|
|
|
|
+ sz = verity_fec_status_table(v, sz, result, maxlen);
|
|
break;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -677,6 +702,7 @@ static void verity_dtr(struct dm_target *ti)
|
|
|
|
|
|
kfree(v->salt);
|
|
kfree(v->salt);
|
|
kfree(v->root_digest);
|
|
kfree(v->root_digest);
|
|
|
|
+ kfree(v->zero_digest);
|
|
|
|
|
|
if (v->tfm)
|
|
if (v->tfm)
|
|
crypto_free_shash(v->tfm);
|
|
crypto_free_shash(v->tfm);
|
|
@@ -689,9 +715,94 @@ static void verity_dtr(struct dm_target *ti)
|
|
if (v->data_dev)
|
|
if (v->data_dev)
|
|
dm_put_device(ti, v->data_dev);
|
|
dm_put_device(ti, v->data_dev);
|
|
|
|
|
|
|
|
+ verity_fec_dtr(v);
|
|
|
|
+
|
|
kfree(v);
|
|
kfree(v);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+static int verity_alloc_zero_digest(struct dm_verity *v)
|
|
|
|
+{
|
|
|
|
+ int r = -ENOMEM;
|
|
|
|
+ struct shash_desc *desc;
|
|
|
|
+ u8 *zero_data;
|
|
|
|
+
|
|
|
|
+ v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL);
|
|
|
|
+
|
|
|
|
+ if (!v->zero_digest)
|
|
|
|
+ return r;
|
|
|
|
+
|
|
|
|
+ desc = kmalloc(v->shash_descsize, GFP_KERNEL);
|
|
|
|
+
|
|
|
|
+ if (!desc)
|
|
|
|
+ return r; /* verity_dtr will free zero_digest */
|
|
|
|
+
|
|
|
|
+ zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL);
|
|
|
|
+
|
|
|
|
+ if (!zero_data)
|
|
|
|
+ goto out;
|
|
|
|
+
|
|
|
|
+ r = verity_hash(v, desc, zero_data, 1 << v->data_dev_block_bits,
|
|
|
|
+ v->zero_digest);
|
|
|
|
+
|
|
|
|
+out:
|
|
|
|
+ kfree(desc);
|
|
|
|
+ kfree(zero_data);
|
|
|
|
+
|
|
|
|
+ return r;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v)
|
|
|
|
+{
|
|
|
|
+ int r;
|
|
|
|
+ unsigned argc;
|
|
|
|
+ struct dm_target *ti = v->ti;
|
|
|
|
+ const char *arg_name;
|
|
|
|
+
|
|
|
|
+ static struct dm_arg _args[] = {
|
|
|
|
+ {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},
|
|
|
|
+ };
|
|
|
|
+
|
|
|
|
+ r = dm_read_arg_group(_args, as, &argc, &ti->error);
|
|
|
|
+ if (r)
|
|
|
|
+ return -EINVAL;
|
|
|
|
+
|
|
|
|
+ if (!argc)
|
|
|
|
+ return 0;
|
|
|
|
+
|
|
|
|
+ do {
|
|
|
|
+ arg_name = dm_shift_arg(as);
|
|
|
|
+ argc--;
|
|
|
|
+
|
|
|
|
+ if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) {
|
|
|
|
+ v->mode = DM_VERITY_MODE_LOGGING;
|
|
|
|
+ continue;
|
|
|
|
+
|
|
|
|
+ } else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) {
|
|
|
|
+ v->mode = DM_VERITY_MODE_RESTART;
|
|
|
|
+ continue;
|
|
|
|
+
|
|
|
|
+ } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) {
|
|
|
|
+ r = verity_alloc_zero_digest(v);
|
|
|
|
+ if (r) {
|
|
|
|
+ ti->error = "Cannot allocate zero digest";
|
|
|
|
+ return r;
|
|
|
|
+ }
|
|
|
|
+ continue;
|
|
|
|
+
|
|
|
|
+ } else if (verity_is_fec_opt_arg(arg_name)) {
|
|
|
|
+ r = verity_fec_parse_opt_args(as, v, &argc, arg_name);
|
|
|
|
+ if (r)
|
|
|
|
+ return r;
|
|
|
|
+ continue;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ ti->error = "Unrecognized verity feature request";
|
|
|
|
+ return -EINVAL;
|
|
|
|
+ } while (argc && !r);
|
|
|
|
+
|
|
|
|
+ return r;
|
|
|
|
+}
|
|
|
|
+
|
|
/*
|
|
/*
|
|
* Target parameters:
|
|
* Target parameters:
|
|
* <version> The current format is version 1.
|
|
* <version> The current format is version 1.
|
|
@@ -710,18 +821,13 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
|
{
|
|
{
|
|
struct dm_verity *v;
|
|
struct dm_verity *v;
|
|
struct dm_arg_set as;
|
|
struct dm_arg_set as;
|
|
- const char *opt_string;
|
|
|
|
- unsigned int num, opt_params;
|
|
|
|
|
|
+ unsigned int num;
|
|
unsigned long long num_ll;
|
|
unsigned long long num_ll;
|
|
int r;
|
|
int r;
|
|
int i;
|
|
int i;
|
|
sector_t hash_position;
|
|
sector_t hash_position;
|
|
char dummy;
|
|
char dummy;
|
|
|
|
|
|
- static struct dm_arg _args[] = {
|
|
|
|
- {0, 1, "Invalid number of feature args"},
|
|
|
|
- };
|
|
|
|
-
|
|
|
|
v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
|
|
v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
|
|
if (!v) {
|
|
if (!v) {
|
|
ti->error = "Cannot allocate verity structure";
|
|
ti->error = "Cannot allocate verity structure";
|
|
@@ -730,6 +836,10 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
|
ti->private = v;
|
|
ti->private = v;
|
|
v->ti = ti;
|
|
v->ti = ti;
|
|
|
|
|
|
|
|
+ r = verity_fec_ctr_alloc(v);
|
|
|
|
+ if (r)
|
|
|
|
+ goto bad;
|
|
|
|
+
|
|
if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) {
|
|
if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) {
|
|
ti->error = "Device must be readonly";
|
|
ti->error = "Device must be readonly";
|
|
r = -EINVAL;
|
|
r = -EINVAL;
|
|
@@ -866,29 +976,9 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
|
as.argc = argc;
|
|
as.argc = argc;
|
|
as.argv = argv;
|
|
as.argv = argv;
|
|
|
|
|
|
- r = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
|
|
|
|
- if (r)
|
|
|
|
|
|
+ r = verity_parse_opt_args(&as, v);
|
|
|
|
+ if (r < 0)
|
|
goto bad;
|
|
goto bad;
|
|
-
|
|
|
|
- while (opt_params) {
|
|
|
|
- opt_params--;
|
|
|
|
- opt_string = dm_shift_arg(&as);
|
|
|
|
- if (!opt_string) {
|
|
|
|
- ti->error = "Not enough feature arguments";
|
|
|
|
- r = -EINVAL;
|
|
|
|
- goto bad;
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- if (!strcasecmp(opt_string, DM_VERITY_OPT_LOGGING))
|
|
|
|
- v->mode = DM_VERITY_MODE_LOGGING;
|
|
|
|
- else if (!strcasecmp(opt_string, DM_VERITY_OPT_RESTART))
|
|
|
|
- v->mode = DM_VERITY_MODE_RESTART;
|
|
|
|
- else {
|
|
|
|
- ti->error = "Invalid feature arguments";
|
|
|
|
- r = -EINVAL;
|
|
|
|
- goto bad;
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
}
|
|
}
|
|
|
|
|
|
v->hash_per_block_bits =
|
|
v->hash_per_block_bits =
|
|
@@ -938,8 +1028,6 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
|
goto bad;
|
|
goto bad;
|
|
}
|
|
}
|
|
|
|
|
|
- ti->per_bio_data_size = roundup(sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2, __alignof__(struct dm_verity_io));
|
|
|
|
-
|
|
|
|
/* WQ_UNBOUND greatly improves performance when running on ramdisk */
|
|
/* WQ_UNBOUND greatly improves performance when running on ramdisk */
|
|
v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
|
|
v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
|
|
if (!v->verify_wq) {
|
|
if (!v->verify_wq) {
|
|
@@ -948,6 +1036,16 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
|
goto bad;
|
|
goto bad;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ ti->per_bio_data_size = sizeof(struct dm_verity_io) +
|
|
|
|
+ v->shash_descsize + v->digest_size * 2;
|
|
|
|
+
|
|
|
|
+ r = verity_fec_ctr(v);
|
|
|
|
+ if (r)
|
|
|
|
+ goto bad;
|
|
|
|
+
|
|
|
|
+ ti->per_bio_data_size = roundup(ti->per_bio_data_size,
|
|
|
|
+ __alignof__(struct dm_verity_io));
|
|
|
|
+
|
|
return 0;
|
|
return 0;
|
|
|
|
|
|
bad:
|
|
bad:
|
|
@@ -958,7 +1056,7 @@ bad:
|
|
|
|
|
|
static struct target_type verity_target = {
|
|
static struct target_type verity_target = {
|
|
.name = "verity",
|
|
.name = "verity",
|
|
- .version = {1, 2, 0},
|
|
|
|
|
|
+ .version = {1, 3, 0},
|
|
.module = THIS_MODULE,
|
|
.module = THIS_MODULE,
|
|
.ctr = verity_ctr,
|
|
.ctr = verity_ctr,
|
|
.dtr = verity_dtr,
|
|
.dtr = verity_dtr,
|