mirror of
https://github.com/aicodix/code.git
synced 2026-04-27 14:30:36 +00:00
improved usability of the new erasure code
This commit is contained in:
Ahmet Inan
parent
8970b8c053
commit
7f492e9d21
3 changed files with 156 additions and 124 deletions
137
cauchy_prime_field_erasure_coding.hh
Normal file
137
cauchy_prime_field_erasure_coding.hh
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
/*
|
||||
Cauchy Prime Field Erasure Coding
|
||||
|
||||
Copyright 2024 Ahmet Inan <inan@aicodix.de>
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace CODE {
|
||||
|
||||
template <typename PF, typename IO, int MAX_LEN>
|
||||
struct CauchyPrimeFieldErasureCoding
|
||||
{
|
||||
PF temp[MAX_LEN];
|
||||
bool used[PF::P];
|
||||
PF row_num, row_den;
|
||||
// $a_{ij} = \frac{1}{x_i + y_j}$
|
||||
PF cauchy_matrix(int i, int j)
|
||||
{
|
||||
PF row(i), col(j);
|
||||
return rcp(row + col);
|
||||
}
|
||||
// $b_{ij} = \frac{\prod_{k=1}^{n}{(x_j + y_k)(x_k + y_i)}}{(x_j + y_i)\prod_{k \ne j}^{n}{(x_j - x_k)}\prod_{k \ne i}^{n}{(y_i - y_k)}}$
|
||||
PF inverse_cauchy_matrix(const int *rows, int i, int j, int n)
|
||||
{
|
||||
#if 0
|
||||
PF row_j(rows[j]), col_i(i);
|
||||
PF prod_xy(1), prod_x(1), prod_y(1);
|
||||
for (int k = 0; k < n; k++) {
|
||||
PF row_k(rows[k]), col_k(k);
|
||||
prod_xy *= (row_j + col_k) * (row_k + col_i);
|
||||
if (k != j)
|
||||
prod_x *= (row_j - row_k);
|
||||
if (k != i)
|
||||
prod_y *= (col_i - col_k);
|
||||
}
|
||||
return prod_xy / ((row_j + col_i) * prod_x * prod_y);
|
||||
#else
|
||||
PF row_j(rows[j]), col_i(i);
|
||||
if (j == 0) {
|
||||
PF num(1), den(1);
|
||||
for (int k = 0, r = 2; k < n; k++, --r) {
|
||||
PF row_k(rows[k]), col_k(k);
|
||||
num = mul(num, add(row_k, col_i));
|
||||
if (k != i)
|
||||
den = mul(den, sub(col_i, col_k));
|
||||
if (!r) {
|
||||
r = 3;
|
||||
num = reduce(num);
|
||||
den = reduce(den);
|
||||
}
|
||||
}
|
||||
row_num = reduce(num);
|
||||
row_den = reduce(den);
|
||||
}
|
||||
PF num(row_num), den(row_den);
|
||||
for (int k = 0, r = 2; k < n; k++, --r) {
|
||||
PF row_k(rows[k]), col_k(k);
|
||||
num = mul(num, add(row_j, col_k));
|
||||
if (k != j)
|
||||
den = mul(den, sub(row_j, row_k));
|
||||
if (!r) {
|
||||
r = 3;
|
||||
num = reduce(num);
|
||||
den = reduce(den);
|
||||
}
|
||||
}
|
||||
num = reduce(num);
|
||||
den = reduce(den);
|
||||
return num / (add(row_j, col_i) * den);
|
||||
#endif
|
||||
}
|
||||
void mac(const IO *a, PF b, int len, bool first, bool last)
|
||||
{
|
||||
if (first) {
|
||||
for (int i = 0; i < len; i++)
|
||||
temp[i] = b * PF(a[i]);
|
||||
} else if (last) {
|
||||
for (int i = 0; i < len; i++)
|
||||
temp[i] = reduce(add(temp[i], b * PF(a[i])));
|
||||
} else {
|
||||
for (int i = 0; i < len; i++)
|
||||
temp[i] = add(temp[i], b * PF(a[i]));
|
||||
}
|
||||
}
|
||||
void mac_sub(IO *c, const IO *a, PF b, int len, bool first, bool last)
|
||||
{
|
||||
int s = a[len], v = PF::P-1;
|
||||
if (first && last) {
|
||||
for (int i = 0; i < len; i++)
|
||||
c[i] = (b * PF(a[i] == s ? v : a[i]))();
|
||||
} else if (first) {
|
||||
for (int i = 0; i < len; i++)
|
||||
temp[i] = b * PF(a[i] == s ? v : a[i]);
|
||||
} else if (last) {
|
||||
for (int i = 0; i < len; i++)
|
||||
c[i] = reduce(add(temp[i], b * PF(a[i] == s ? v : a[i])))();
|
||||
} else {
|
||||
for (int i = 0; i < len; i++)
|
||||
temp[i] = add(temp[i], b * PF(a[i] == s ? v : a[i]));
|
||||
}
|
||||
}
|
||||
int find_unused(int block_len)
|
||||
{
|
||||
for (int i = 0; i < int(PF::P); ++i)
|
||||
used[i] = false;
|
||||
for (int i = 0; i < block_len; ++i)
|
||||
used[temp[i]()] = true;
|
||||
int s = 0;
|
||||
while (used[s])
|
||||
++s;
|
||||
return s;
|
||||
}
|
||||
void encode(const IO *data, IO *block, int block_id, int block_len, int block_cnt)
|
||||
{
|
||||
assert(block_id >= block_cnt && block_id < int(PF::P) / 2);
|
||||
assert(block_len < int(PF::P-1) && block_len <= MAX_LEN);
|
||||
for (int k = 0; k < block_cnt; k++) {
|
||||
PF a_ik = cauchy_matrix(block_id, k);
|
||||
mac(data + block_len * k, a_ik, block_len, !k, k == block_cnt - 1);
|
||||
}
|
||||
int sub = find_unused(block_len);
|
||||
for (int i = 0; i < block_len; ++i)
|
||||
block[i] = temp[i]() == PF::P-1 ? sub : temp[i]();
|
||||
block[block_len] = sub;
|
||||
}
|
||||
void decode(IO *data, const IO *blocks, const int *block_ids, int block_idx, int block_len, int block_cnt)
|
||||
{
|
||||
for (int k = 0; k < block_cnt; k++) {
|
||||
PF b_ik = inverse_cauchy_matrix(block_ids, block_idx, k, block_cnt);
|
||||
mac_sub(data, blocks + (block_len+1) * k, b_ik, block_len, !k, k == block_cnt - 1);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
/*
|
||||
Cauchy Reed Solomon Erasure Coding
|
||||
|
||||
Copyright 2024 Ahmet Inan <inan@aicodix.de>
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace CODE {
|
||||
|
||||
template <typename PF>
|
||||
struct CauchyReedSolomonErasureCoding2
|
||||
{
|
||||
PF row_num, row_den;
|
||||
// $a_{ij} = \frac{1}{x_i + y_j}$
|
||||
__attribute__((flatten))
|
||||
PF cauchy_matrix(int i, int j)
|
||||
{
|
||||
PF row(i), col(j);
|
||||
return rcp(row + col);
|
||||
}
|
||||
// $b_{ij} = \frac{\prod_{k=1}^{n}{(x_j + y_k)(x_k + y_i)}}{(x_j + y_i)\prod_{k \ne j}^{n}{(x_j - x_k)}\prod_{k \ne i}^{n}{(y_i - y_k)}}$
|
||||
__attribute__((flatten))
|
||||
PF inverse_cauchy_matrix(const PF *rows, int i, int j, int n)
|
||||
{
|
||||
#if 0
|
||||
PF col_i(i);
|
||||
PF prod_xy(1), prod_x(1), prod_y(1);
|
||||
for (int k = 0; k < n; k++) {
|
||||
PF col_k(k);
|
||||
prod_xy *= (rows[j] + col_k) * (rows[k] + col_i);
|
||||
if (k != j)
|
||||
prod_x *= (rows[j] - rows[k]);
|
||||
if (k != i)
|
||||
prod_y *= (col_i - col_k);
|
||||
}
|
||||
return prod_xy / ((rows[j] + col_i) * prod_x * prod_y);
|
||||
#else
|
||||
PF col_i(i);
|
||||
if (j == 0) {
|
||||
PF num(1), den(1);
|
||||
for (int k = 0, r = 2; k < n; k++, --r) {
|
||||
PF col_k(k);
|
||||
num = mul(num, add(rows[k], col_i));
|
||||
if (k != i)
|
||||
den = mul(den, sub(col_i, col_k));
|
||||
if (!r) {
|
||||
r = 3;
|
||||
num = reduce(num);
|
||||
den = reduce(den);
|
||||
}
|
||||
}
|
||||
row_num = reduce(num);
|
||||
row_den = reduce(den);
|
||||
}
|
||||
PF num(row_num), den(row_den);
|
||||
for (int k = 0, r = 2; k < n; k++, --r) {
|
||||
PF col_k(k);
|
||||
num = mul(num, add(rows[j], col_k));
|
||||
if (k != j)
|
||||
den = mul(den, sub(rows[j], rows[k]));
|
||||
if (!r) {
|
||||
r = 3;
|
||||
num = reduce(num);
|
||||
den = reduce(den);
|
||||
}
|
||||
}
|
||||
num = reduce(num);
|
||||
den = reduce(den);
|
||||
return num / (add(rows[j], col_i) * den);
|
||||
#endif
|
||||
}
|
||||
__attribute__((flatten))
|
||||
static inline void multiply_accumulate(PF *c, const PF *a, PF b, int len, bool first, bool last)
|
||||
{
|
||||
if (first) {
|
||||
for (int i = 0; i < len; i++)
|
||||
c[i] = b * a[i];
|
||||
} else if (last) {
|
||||
for (int i = 0; i < len; i++)
|
||||
c[i] = reduce(add(c[i], b * a[i]));
|
||||
} else {
|
||||
for (int i = 0; i < len; i++)
|
||||
c[i] = add(c[i], b * a[i]);
|
||||
}
|
||||
}
|
||||
void encode(const PF *data, PF *block, int block_id, int block_len, int block_cnt)
|
||||
{
|
||||
assert(block_id >= block_cnt && block_id < int(PF::P) / 2);
|
||||
for (int k = 0; k < block_cnt; k++) {
|
||||
PF a_ik = cauchy_matrix(block_id, k);
|
||||
multiply_accumulate(block, data + block_len * k, a_ik, block_len, !k, k == block_cnt - 1);
|
||||
}
|
||||
}
|
||||
void decode(PF *data, const PF *blocks, const PF *block_ids, int block_idx, int block_len, int block_cnt)
|
||||
{
|
||||
for (int k = 0; k < block_cnt; k++) {
|
||||
PF b_ik = inverse_cauchy_matrix(block_ids, block_idx, k, block_cnt);
|
||||
multiply_accumulate(data, blocks + block_len * k, b_ik, block_len, !k, k == block_cnt - 1);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
Regression Test for the second Cauchy Reed Solomon Encoder and Decoder
|
||||
Regression Test for the Cauchy Prime Field Encoder and Decoder
|
||||
|
||||
Copyright 2024 Ahmet Inan <inan@aicodix.de>
|
||||
*/
|
||||
|
|
@ -11,20 +11,20 @@ Copyright 2024 Ahmet Inan <inan@aicodix.de>
|
|||
#include <iostream>
|
||||
#include <functional>
|
||||
#include "prime_field.hh"
|
||||
#include "cauchy_reed_solomon_erasure_coding2.hh"
|
||||
#include "cauchy_prime_field_erasure_coding.hh"
|
||||
|
||||
template <typename TYPE, TYPE PRIME>
|
||||
void crs_test(int trials)
|
||||
template <typename PF, typename IO>
|
||||
void cpf_test(int trials)
|
||||
{
|
||||
int value_bits = log2(PRIME);
|
||||
int value_bytes = value_bits / 8;
|
||||
typedef CODE::PrimeField<TYPE, PRIME> PF;
|
||||
CODE::CauchyReedSolomonErasureCoding2<PF> crs;
|
||||
int value_bytes = sizeof(IO);
|
||||
int value_bits = value_bytes * 8;
|
||||
const int MAX_LEN = std::min<int>(PF::P - 2, 1024);
|
||||
CODE::CauchyPrimeFieldErasureCoding<PF, IO, MAX_LEN> crs;
|
||||
std::random_device rd;
|
||||
std::default_random_engine generator(rd());
|
||||
typedef std::uniform_int_distribution<int> distribution;
|
||||
auto rnd_cnt = std::bind(distribution(1, std::min<int>(PF::P / 4, 256)), generator);
|
||||
auto rnd_len = std::bind(distribution(1, 1 << 10), generator);
|
||||
auto rnd_len = std::bind(distribution(1, MAX_LEN), generator);
|
||||
auto rnd_dat = std::bind(distribution(0, (1 << value_bits) - 1), generator);
|
||||
while (--trials) {
|
||||
int block_count = rnd_cnt();
|
||||
|
|
@ -33,21 +33,21 @@ void crs_test(int trials)
|
|||
int block_bytes = block_values * value_bytes;
|
||||
int data_values = block_count * block_values;
|
||||
int data_bytes = data_values * value_bytes;
|
||||
PF *orig = new PF[data_values];
|
||||
PF *data = new PF[data_values];
|
||||
PF *blocks = new PF[data_values];
|
||||
IO *orig = new IO[data_values];
|
||||
IO *data = new IO[data_values];
|
||||
IO *blocks = new IO[data_values+block_count];
|
||||
for (int i = 0; i < data_values; ++i)
|
||||
orig[i] = PF(rnd_dat());
|
||||
auto identifiers = new PF[identifiers_total];
|
||||
orig[i] = rnd_dat();
|
||||
auto identifiers = new int[identifiers_total];
|
||||
for (int i = 0; i < identifiers_total; ++i)
|
||||
identifiers[i] = PF(block_count + i);
|
||||
identifiers[i] = block_count + i;
|
||||
for (int i = 0; i < block_count; i++) {
|
||||
std::uniform_int_distribution<int> hat(i, identifiers_total - 1);
|
||||
std::swap(identifiers[i], identifiers[hat(generator)]);
|
||||
}
|
||||
auto enc_start = std::chrono::system_clock::now();
|
||||
for (int i = 0; i < block_count; ++i)
|
||||
crs.encode(orig, blocks + block_values * i, identifiers[i](), block_values, block_count);
|
||||
crs.encode(orig, blocks + (block_values+1) * i, identifiers[i], block_values, block_count);
|
||||
auto enc_end = std::chrono::system_clock::now();
|
||||
auto enc_usec = std::chrono::duration_cast<std::chrono::microseconds>(enc_end - enc_start);
|
||||
double enc_mbs = double(data_bytes) / enc_usec.count();
|
||||
|
|
@ -70,12 +70,12 @@ void crs_test(int trials)
|
|||
int main()
|
||||
{
|
||||
if (1) {
|
||||
crs_test<uint32_t, 257>(200);
|
||||
cpf_test<CODE::PrimeField<uint32_t, 257>, uint8_t>(200);
|
||||
}
|
||||
if (1) {
|
||||
crs_test<uint64_t, 65537>(100);
|
||||
cpf_test<CODE::PrimeField<uint64_t, 65537>, uint16_t>(100);
|
||||
}
|
||||
std::cerr << "Cauchy Reed Solomon Two regression test passed!" << std::endl;
|
||||
std::cerr << "Cauchy prime field regression test passed!" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue