From c7f54515015744f08a847be1f356de310290d5de Mon Sep 17 00:00:00 2001 From: Thomas Schoebel-Theuer Date: Tue, 26 Mar 2019 10:57:33 +0100 Subject: [PATCH] infra: prepare multiple digests --- kernel/lib_log.c | 42 ++++++++++++++++++++++++++++++---- kernel/lib_log.h | 2 ++ kernel/mars.h | 19 ++++++++++++++-- kernel/mars_copy.c | 14 ++++++++---- kernel/mars_generic.c | 48 +++++++++++++++++++++++++++++++++------ kernel/sy_old/mars_main.c | 5 +++- 6 files changed, 111 insertions(+), 19 deletions(-) diff --git a/kernel/lib_log.c b/kernel/lib_log.c index dea6f587..1aaad7c4 100644 --- a/kernel/lib_log.c +++ b/kernel/lib_log.c @@ -321,6 +321,8 @@ bool log_finalize(struct log_status *logst, int len, void (*endio)(void *private int restlen; int nr_cb; int crc; + __u32 check_flags; + __u16 crc_flags; bool ok = false; CHECK_PTR(mref, err); @@ -342,14 +344,27 @@ bool log_finalize(struct log_status *logst, int len, void (*endio)(void *private data = mref->ref_data; crc = 0; + check_flags = 0; if (logst->do_crc) { unsigned char checksum[MARS_DIGEST_SIZE]; - mars_digest(checksum, data + logst->payload_offset, len); + check_flags |= + mars_digest(usable_digest_mask, + &used_log_digest, + checksum, + data + logst->payload_offset, len); /* FIXME: extend to MARS_DIGEST_SIZE */ crc = *(int*)checksum; } + /* + * We have only 16 flag bits in the traditional + * logfile format, which is in production over + * years. To remain compatible, we strip off + * non-checksum related bits. + */ + crc_flags = check_flags >> _MREF_CHKSUM_MD5_OLD; + /* Correct the length in the header. */ offset = logst->reallen_offset; @@ -362,7 +377,7 @@ bool log_finalize(struct log_status *logst, int len, void (*endio)(void *private DATA_PUT(data, offset, crc); DATA_PUT(data, offset, (char)1); // valid_flag copy DATA_PUT(data, offset, (char)0); // spare - DATA_PUT(data, offset, (short)0); // spare + DATA_PUT(data, offset, crc_flags); DATA_PUT(data, offset, logst->seq_nr + 1); get_lamport(NULL, &now); // when the log entry was ready. DATA_PUT(data, offset, now.tv_sec); @@ -565,6 +580,7 @@ int log_scan(void *buf, short total_len; long long end_magic; char valid_copy; + __u32 check_flags; int restlen = 0; int found_offset; @@ -638,8 +654,9 @@ int log_scan(void *buf, } // skip spares - offset += 3; + offset += 1; + DATA_GET(buf, offset, lh->l_crc_flags); DATA_GET(buf, offset, lh->l_seq_nr); DATA_GET(buf, offset, lh->l_written.tv_sec); DATA_GET(buf, offset, lh->l_written.tv_nsec); @@ -652,10 +669,27 @@ int log_scan(void *buf, } *seq_nr = lh->l_seq_nr; + /* + * We have only 16 flag bits in the traditional + * logfile format, which is in production over + * years. To remain compatible, we strip off + * non-checksum related bits. + */ + check_flags = + (((__u32)lh->l_crc_flags) << _MREF_CHKSUM_MD5_OLD) & + available_digest_mask; + + /* compatibility with old logfiles during upgrade */ + if (!check_flags) + check_flags = MREF_CHKSUM_MD5_OLD; + if (lh->l_crc) { unsigned char checksum[MARS_DIGEST_SIZE]; - mars_digest(checksum, buf + found_offset, lh->l_len); + mars_digest(check_flags, + &used_log_digest, + checksum, + buf + found_offset, lh->l_len); /* FIXME: extend to MARS_DIGEST_SIZE */ if (unlikely(*(int*)checksum != lh->l_crc)) { MARS_ERR(SCAN_TXT "data checksumming mismatch, length = %d\n", SCAN_PAR, lh->l_len); diff --git a/kernel/lib_log.h b/kernel/lib_log.h index 385b29a8..a92fde9b 100644 --- a/kernel/lib_log.h +++ b/kernel/lib_log.h @@ -54,7 +54,9 @@ struct log_header_v1 { short l_len; short l_code; unsigned int l_seq_nr; + /* TODO: extend to 64 bit */ int l_crc; + __u16 l_crc_flags; }; #define FORMAT_VERSION 1 // version of disk format, currently there is no other one diff --git a/kernel/mars.h b/kernel/mars.h index c0078f73..e4d5fe17 100644 --- a/kernel/mars.h +++ b/kernel/mars.h @@ -162,6 +162,7 @@ enum _MREF_FLAGS { _MREF_NODATA, /* only useful for chksum reading */ /* Checksum bits, starting at the upper half */ _MREF_CHKSUM_MD5_OLD = 16, + _MREF_CHKSUM_LAST, }; #define MREF_UPTODATE (1UL << _MREF_UPTODATE) @@ -172,7 +173,9 @@ enum _MREF_FLAGS { #define MREF_SKIP_SYNC (1UL << _MREF_SKIP_SYNC) #define MREF_NODATA (1UL << _MREF_NODATA) #define MREF_CHKSUM_MD5_OLD (1UL << _MREF_CHKSUM_MD5_OLD) -#define MREF_CHKSUM_ANY (MREF_CHKSUM_MD5_OLD) +#define MREF_CHKSUM_LAST (1UL << _MREF_CHKSUM_LAST) +#define MREF_CHKSUM_ANY (MREF_CHKSUM_MD5_OLD | \ + MREF_CHKSUM_LAST) #define MREF_OBJECT(OBJTYPE) \ CALLBACK_OBJECT(OBJTYPE); \ @@ -487,7 +490,19 @@ static inline void unuse_fake_mm(void) {} /* Crypto / digest stuff */ -extern void mars_digest(unsigned char *digest, void *data, int len); +extern __u32 available_digest_mask; +extern __u32 usable_digest_mask; +extern __u32 used_log_digest; +extern __u32 used_net_digest; + +extern __u32 disabled_log_digests; +extern __u32 disabled_net_digests; + +extern __u32 mars_digest(__u32 digest_flags, + __u32 *used_flags, + void *digest, + void *data, int len); + extern void mref_checksum(struct mref_object *mref); ///////////////////////////////////////////////////////////////////////// diff --git a/kernel/mars_copy.c b/kernel/mars_copy.c index f91ce2ad..4472543e 100644 --- a/kernel/mars_copy.c +++ b/kernel/mars_copy.c @@ -376,11 +376,15 @@ void _update_percent(struct copy_brick *brick, bool force) } static inline -__u32 _make_flags(bool verify_mode) +__u32 _make_flags(bool verify_mode, bool is_local) { if (!verify_mode) return 0; - return MREF_NODATA | MREF_CHKSUM_ANY; + + if (is_local) + return available_digest_mask | MREF_NODATA; + + return usable_digest_mask | MREF_NODATA; } @@ -452,7 +456,7 @@ restart: status = _make_mref(brick, index, 0, NULL, pos, brick->copy_end, - _make_flags(brick->verify_mode)); + _make_flags(brick->verify_mode, false)); if (unlikely(status < 0)) { MARS_DBG("status = %d\n", status); progress = status; @@ -469,7 +473,7 @@ restart: case COPY_STATE_START2: status = _make_mref(brick, index, 1, NULL, pos, brick->copy_end, - _make_flags(true)); + _make_flags(true, true)); if (unlikely(status < 0)) { MARS_DBG("status = %d\n", status); progress = status; @@ -534,7 +538,7 @@ restart: _clear_mref(brick, index, 0); status = _make_mref(brick, index, 0, NULL, pos, brick->copy_end, - _make_flags(false)); + _make_flags(false, false)); if (unlikely(status < 0)) { MARS_DBG("status = %d\n", status); progress = status; diff --git a/kernel/mars_generic.c b/kernel/mars_generic.c index 6b2ae596..0b6756e3 100644 --- a/kernel/mars_generic.c +++ b/kernel/mars_generic.c @@ -144,9 +144,14 @@ EXPORT_SYMBOL_GPL(mars_lamport_time_meta); #define MD5_DIGEST_SIZE 16 +__u32 available_digest_mask = MREF_CHKSUM_MD5_OLD; +__u32 usable_digest_mask = MREF_CHKSUM_MD5_OLD; +__u32 used_log_digest = 0; +__u32 used_net_digest = 0; + #ifdef MARS_HAS_NEW_CRYPTO -/* Nor now, use shash. +/* For now, use shash. * Later, asynchronous support should be added for full exploitation * of crypto hardware. */ @@ -164,7 +169,8 @@ struct mars_sdesc { * digest bytes up to MARS_DIGEST_SIZE are not exploited * in this version. */ -void mars_digest(unsigned char *digest, void *data, int len) +static +void md5_old_digest(void *digest, void *data, int len) { int size = sizeof(struct mars_sdesc) + crypto_shash_descsize(md5_tfm); struct mars_sdesc *sdesc = brick_mem_alloc(size); @@ -175,14 +181,30 @@ void mars_digest(unsigned char *digest, void *data, int len) memset(digest, 0, MARS_DIGEST_SIZE); status = crypto_shash_digest(&sdesc->shash, data, len, digest); - if (unlikely(status < 0)) - MARS_ERR("cannot calculate cksum on %p len=%d, status=%d\n", + if (unlikely(status < 0)) { + MARS_ERR("cannot calculate md5 chksum on %p len=%d, status=%d\n", data, len, status); + memset(digest, 0, MARS_DIGEST_SIZE); + } brick_mem_free(sdesc); } +__u32 mars_digest(__u32 digest_flags, + __u32 *used_flags, + void *digest, + void *data, int len) +{ + digest_flags &= usable_digest_mask; + + /* always fallback to old md5 regardless of digest_flags */ + md5_old_digest(digest, data, len); + if (used_flags) + *used_flags = MREF_CHKSUM_MD5_OLD; + return MREF_CHKSUM_MD5_OLD; +} + static int init_mars_digest(void) { @@ -224,7 +246,10 @@ void exit_mars_digest(void) static struct crypto_hash *mars_tfm[OBSOLETE_TFM_MAX]; static struct semaphore tfm_sem[OBSOLETE_TFM_MAX]; -void mars_digest(unsigned char *digest, void *data, int len) +__u32 mars_digest(__u32 digest_flags, + __u32 *used_flags, + void *digest, + void *data, int len) { static unsigned int round_robin = 0; unsigned int i = round_robin++ % OBSOLETE_TFM_MAX; @@ -236,7 +261,6 @@ void mars_digest(unsigned char *digest, void *data, int len) memset(digest, 0, MARS_DIGEST_SIZE); - // TODO: use per-thread instance, omit locking down(&tfm_sem[i]); crypto_hash_init(&desc); @@ -245,6 +269,9 @@ void mars_digest(unsigned char *digest, void *data, int len) crypto_hash_update(&desc, &sg, sg.length); crypto_hash_final(&desc, digest); up(&tfm_sem[i]); + if (used_flags) + *used_flags = MREF_CHKSUM_MD5_OLD; + return MREF_CHKSUM_MD5_OLD; } #endif /* MARS_HAS_NEW_CRYPTO */ @@ -252,12 +279,19 @@ void mars_digest(unsigned char *digest, void *data, int len) void mref_checksum(struct mref_object *mref) { unsigned char checksum[MARS_DIGEST_SIZE]; + __u32 flags; int len; if (!(mref->ref_flags & MREF_CHKSUM_ANY) || !mref->ref_data) return; - mars_digest(checksum, mref->ref_data, mref->ref_len); + flags = + mars_digest(mref->ref_flags & usable_digest_mask, + &used_net_digest, + checksum, + mref->ref_data, mref->ref_len); + + mref->ref_flags = (mref->ref_flags & ~MREF_CHKSUM_ANY) | flags; len = sizeof(mref->ref_checksum); if (len > MARS_DIGEST_SIZE) diff --git a/kernel/sy_old/mars_main.c b/kernel/sy_old/mars_main.c index 31cb74e1..9cddc372 100644 --- a/kernel/sy_old/mars_main.c +++ b/kernel/sy_old/mars_main.c @@ -1312,7 +1312,10 @@ int _update_version_link(struct mars_rotate *rot, struct trans_logger_info *inf) MARS_DBG("data = '%s' len = %d\n", data, len); - mars_digest(digest, data, len); + mars_digest(MREF_CHKSUM_MD5_OLD, + NULL, + digest, + data, len); len = 0; /* Maintain compatibilty with old behaviour */