From 7f492e9d21381748f5f0a7c5ffd285b6113c47ec Mon Sep 17 00:00:00 2001 From: Ahmet Inan Date: Tue, 26 Mar 2024 16:46:43 +0100 Subject: [PATCH] improved usability of the new erasure code --- cauchy_prime_field_erasure_coding.hh | 137 ++++++++++++++++++ cauchy_reed_solomon_erasure_coding2.hh | 105 -------------- ...ression_test.cc => cpf_regression_test.cc} | 38 ++--- 3 files changed, 156 insertions(+), 124 deletions(-) create mode 100644 cauchy_prime_field_erasure_coding.hh delete mode 100644 cauchy_reed_solomon_erasure_coding2.hh rename tests/{crs2_regression_test.cc => cpf_regression_test.cc} (69%) diff --git a/cauchy_prime_field_erasure_coding.hh b/cauchy_prime_field_erasure_coding.hh new file mode 100644 index 0000000..eeb38a0 --- /dev/null +++ b/cauchy_prime_field_erasure_coding.hh @@ -0,0 +1,137 @@ +/* +Cauchy Prime Field Erasure Coding + +Copyright 2024 Ahmet Inan +*/ + +#pragma once + +namespace CODE { + +template +struct CauchyPrimeFieldErasureCoding +{ + PF temp[MAX_LEN]; + bool used[PF::P]; + PF row_num, row_den; + // $a_{ij} = \frac{1}{x_i + y_j}$ + PF cauchy_matrix(int i, int j) + { + PF row(i), col(j); + return rcp(row + col); + } + // $b_{ij} = \frac{\prod_{k=1}^{n}{(x_j + y_k)(x_k + y_i)}}{(x_j + y_i)\prod_{k \ne j}^{n}{(x_j - x_k)}\prod_{k \ne i}^{n}{(y_i - y_k)}}$ + PF inverse_cauchy_matrix(const int *rows, int i, int j, int n) + { +#if 0 + PF row_j(rows[j]), col_i(i); + PF prod_xy(1), prod_x(1), prod_y(1); + for (int k = 0; k < n; k++) { + PF row_k(rows[k]), col_k(k); + prod_xy *= (row_j + col_k) * (row_k + col_i); + if (k != j) + prod_x *= (row_j - row_k); + if (k != i) + prod_y *= (col_i - col_k); + } + return prod_xy / ((row_j + col_i) * prod_x * prod_y); +#else + PF row_j(rows[j]), col_i(i); + if (j == 0) { + PF num(1), den(1); + for (int k = 0, r = 2; k < n; k++, --r) { + PF row_k(rows[k]), col_k(k); + num = mul(num, add(row_k, col_i)); + if (k != i) + den = mul(den, sub(col_i, col_k)); + if (!r) { + r = 3; + num = reduce(num); + den = reduce(den); + } + } + row_num = reduce(num); + row_den = reduce(den); + } + PF num(row_num), den(row_den); + for (int k = 0, r = 2; k < n; k++, --r) { + PF row_k(rows[k]), col_k(k); + num = mul(num, add(row_j, col_k)); + if (k != j) + den = mul(den, sub(row_j, row_k)); + if (!r) { + r = 3; + num = reduce(num); + den = reduce(den); + } + } + num = reduce(num); + den = reduce(den); + return num / (add(row_j, col_i) * den); +#endif + } + void mac(const IO *a, PF b, int len, bool first, bool last) + { + if (first) { + for (int i = 0; i < len; i++) + temp[i] = b * PF(a[i]); + } else if (last) { + for (int i = 0; i < len; i++) + temp[i] = reduce(add(temp[i], b * PF(a[i]))); + } else { + for (int i = 0; i < len; i++) + temp[i] = add(temp[i], b * PF(a[i])); + } + } + void mac_sub(IO *c, const IO *a, PF b, int len, bool first, bool last) + { + int s = a[len], v = PF::P-1; + if (first && last) { + for (int i = 0; i < len; i++) + c[i] = (b * PF(a[i] == s ? v : a[i]))(); + } else if (first) { + for (int i = 0; i < len; i++) + temp[i] = b * PF(a[i] == s ? v : a[i]); + } else if (last) { + for (int i = 0; i < len; i++) + c[i] = reduce(add(temp[i], b * PF(a[i] == s ? v : a[i])))(); + } else { + for (int i = 0; i < len; i++) + temp[i] = add(temp[i], b * PF(a[i] == s ? v : a[i])); + } + } + int find_unused(int block_len) + { + for (int i = 0; i < int(PF::P); ++i) + used[i] = false; + for (int i = 0; i < block_len; ++i) + used[temp[i]()] = true; + int s = 0; + while (used[s]) + ++s; + return s; + } + void encode(const IO *data, IO *block, int block_id, int block_len, int block_cnt) + { + assert(block_id >= block_cnt && block_id < int(PF::P) / 2); + assert(block_len < int(PF::P-1) && block_len <= MAX_LEN); + for (int k = 0; k < block_cnt; k++) { + PF a_ik = cauchy_matrix(block_id, k); + mac(data + block_len * k, a_ik, block_len, !k, k == block_cnt - 1); + } + int sub = find_unused(block_len); + for (int i = 0; i < block_len; ++i) + block[i] = temp[i]() == PF::P-1 ? sub : temp[i](); + block[block_len] = sub; + } + void decode(IO *data, const IO *blocks, const int *block_ids, int block_idx, int block_len, int block_cnt) + { + for (int k = 0; k < block_cnt; k++) { + PF b_ik = inverse_cauchy_matrix(block_ids, block_idx, k, block_cnt); + mac_sub(data, blocks + (block_len+1) * k, b_ik, block_len, !k, k == block_cnt - 1); + } + } +}; + +} + diff --git a/cauchy_reed_solomon_erasure_coding2.hh b/cauchy_reed_solomon_erasure_coding2.hh deleted file mode 100644 index d4c31ee..0000000 --- a/cauchy_reed_solomon_erasure_coding2.hh +++ /dev/null @@ -1,105 +0,0 @@ -/* -Cauchy Reed Solomon Erasure Coding - -Copyright 2024 Ahmet Inan -*/ - -#pragma once - -namespace CODE { - -template -struct CauchyReedSolomonErasureCoding2 -{ - PF row_num, row_den; - // $a_{ij} = \frac{1}{x_i + y_j}$ - __attribute__((flatten)) - PF cauchy_matrix(int i, int j) - { - PF row(i), col(j); - return rcp(row + col); - } - // $b_{ij} = \frac{\prod_{k=1}^{n}{(x_j + y_k)(x_k + y_i)}}{(x_j + y_i)\prod_{k \ne j}^{n}{(x_j - x_k)}\prod_{k \ne i}^{n}{(y_i - y_k)}}$ - __attribute__((flatten)) - PF inverse_cauchy_matrix(const PF *rows, int i, int j, int n) - { -#if 0 - PF col_i(i); - PF prod_xy(1), prod_x(1), prod_y(1); - for (int k = 0; k < n; k++) { - PF col_k(k); - prod_xy *= (rows[j] + col_k) * (rows[k] + col_i); - if (k != j) - prod_x *= (rows[j] - rows[k]); - if (k != i) - prod_y *= (col_i - col_k); - } - return prod_xy / ((rows[j] + col_i) * prod_x * prod_y); -#else - PF col_i(i); - if (j == 0) { - PF num(1), den(1); - for (int k = 0, r = 2; k < n; k++, --r) { - PF col_k(k); - num = mul(num, add(rows[k], col_i)); - if (k != i) - den = mul(den, sub(col_i, col_k)); - if (!r) { - r = 3; - num = reduce(num); - den = reduce(den); - } - } - row_num = reduce(num); - row_den = reduce(den); - } - PF num(row_num), den(row_den); - for (int k = 0, r = 2; k < n; k++, --r) { - PF col_k(k); - num = mul(num, add(rows[j], col_k)); - if (k != j) - den = mul(den, sub(rows[j], rows[k])); - if (!r) { - r = 3; - num = reduce(num); - den = reduce(den); - } - } - num = reduce(num); - den = reduce(den); - return num / (add(rows[j], col_i) * den); -#endif - } - __attribute__((flatten)) - static inline void multiply_accumulate(PF *c, const PF *a, PF b, int len, bool first, bool last) - { - if (first) { - for (int i = 0; i < len; i++) - c[i] = b * a[i]; - } else if (last) { - for (int i = 0; i < len; i++) - c[i] = reduce(add(c[i], b * a[i])); - } else { - for (int i = 0; i < len; i++) - c[i] = add(c[i], b * a[i]); - } - } - void encode(const PF *data, PF *block, int block_id, int block_len, int block_cnt) - { - assert(block_id >= block_cnt && block_id < int(PF::P) / 2); - for (int k = 0; k < block_cnt; k++) { - PF a_ik = cauchy_matrix(block_id, k); - multiply_accumulate(block, data + block_len * k, a_ik, block_len, !k, k == block_cnt - 1); - } - } - void decode(PF *data, const PF *blocks, const PF *block_ids, int block_idx, int block_len, int block_cnt) - { - for (int k = 0; k < block_cnt; k++) { - PF b_ik = inverse_cauchy_matrix(block_ids, block_idx, k, block_cnt); - multiply_accumulate(data, blocks + block_len * k, b_ik, block_len, !k, k == block_cnt - 1); - } - } -}; - -} - diff --git a/tests/crs2_regression_test.cc b/tests/cpf_regression_test.cc similarity index 69% rename from tests/crs2_regression_test.cc rename to tests/cpf_regression_test.cc index 250ca95..cc7fe92 100644 --- a/tests/crs2_regression_test.cc +++ b/tests/cpf_regression_test.cc @@ -1,5 +1,5 @@ /* -Regression Test for the second Cauchy Reed Solomon Encoder and Decoder +Regression Test for the Cauchy Prime Field Encoder and Decoder Copyright 2024 Ahmet Inan */ @@ -11,20 +11,20 @@ Copyright 2024 Ahmet Inan #include #include #include "prime_field.hh" -#include "cauchy_reed_solomon_erasure_coding2.hh" +#include "cauchy_prime_field_erasure_coding.hh" -template -void crs_test(int trials) +template +void cpf_test(int trials) { - int value_bits = log2(PRIME); - int value_bytes = value_bits / 8; - typedef CODE::PrimeField PF; - CODE::CauchyReedSolomonErasureCoding2 crs; + int value_bytes = sizeof(IO); + int value_bits = value_bytes * 8; + const int MAX_LEN = std::min(PF::P - 2, 1024); + CODE::CauchyPrimeFieldErasureCoding crs; std::random_device rd; std::default_random_engine generator(rd()); typedef std::uniform_int_distribution distribution; auto rnd_cnt = std::bind(distribution(1, std::min(PF::P / 4, 256)), generator); - auto rnd_len = std::bind(distribution(1, 1 << 10), generator); + auto rnd_len = std::bind(distribution(1, MAX_LEN), generator); auto rnd_dat = std::bind(distribution(0, (1 << value_bits) - 1), generator); while (--trials) { int block_count = rnd_cnt(); @@ -33,21 +33,21 @@ void crs_test(int trials) int block_bytes = block_values * value_bytes; int data_values = block_count * block_values; int data_bytes = data_values * value_bytes; - PF *orig = new PF[data_values]; - PF *data = new PF[data_values]; - PF *blocks = new PF[data_values]; + IO *orig = new IO[data_values]; + IO *data = new IO[data_values]; + IO *blocks = new IO[data_values+block_count]; for (int i = 0; i < data_values; ++i) - orig[i] = PF(rnd_dat()); - auto identifiers = new PF[identifiers_total]; + orig[i] = rnd_dat(); + auto identifiers = new int[identifiers_total]; for (int i = 0; i < identifiers_total; ++i) - identifiers[i] = PF(block_count + i); + identifiers[i] = block_count + i; for (int i = 0; i < block_count; i++) { std::uniform_int_distribution hat(i, identifiers_total - 1); std::swap(identifiers[i], identifiers[hat(generator)]); } auto enc_start = std::chrono::system_clock::now(); for (int i = 0; i < block_count; ++i) - crs.encode(orig, blocks + block_values * i, identifiers[i](), block_values, block_count); + crs.encode(orig, blocks + (block_values+1) * i, identifiers[i], block_values, block_count); auto enc_end = std::chrono::system_clock::now(); auto enc_usec = std::chrono::duration_cast(enc_end - enc_start); double enc_mbs = double(data_bytes) / enc_usec.count(); @@ -70,12 +70,12 @@ void crs_test(int trials) int main() { if (1) { - crs_test(200); + cpf_test, uint8_t>(200); } if (1) { - crs_test(100); + cpf_test, uint16_t>(100); } - std::cerr << "Cauchy Reed Solomon Two regression test passed!" << std::endl; + std::cerr << "Cauchy prime field regression test passed!" << std::endl; return 0; }