競技プログラミング用 C++ ライブラリ
components/polynomial_gcd_blocked.hpp · GitHub で見る
共通ページ · 軽量実装なし
polynomial_gcd.hpp が使う Linux x86_64 / GCC 用の内部部品です。通常は公開関数 polynomial_gcd()、polynomial_extended_gcd()、polynomial_inv_mod() を使います。専用経路を呼ぶ前に fps_detail::NTT::vectorized() を確認します。他の環境では既存の Half-GCD へ戻ります。
| 操作 | 内容・返り値 | 計算量 |
|---|---|---|
HalfGcd::words(n) |
必要な作業領域のワード数 | \(O(1)\) |
HalfGcd(arena,n) |
変換表と作業領域を準備 | \(O(n)\) |
matrix(k,room) |
変換行列用の領域を取得 | \(O(1)\) |
jump(a,n,b,m,k,out,need) |
互除法をまとめた変換行列を計算 | \(O(n\log^2 n)\) |
native_inv_mod(a,n,b,m,out) |
法多項式での逆元。存在すれば true | \(O(n\log^2 n)\) |
native_gcd_matrix(a,n,b,m,need,sink) |
変換行列が有効な間に sink を呼ぶ | \(O(n\log^2 n)\) |
fps_detail::blocked_poly::gcd_detail::HalfGcd は次数ごとの jump を求めます。HalfGcd::words(n) ワードの Arena を用意し、HalfGcd(arena,n) で変換表と LIFO の作業領域を確保します。作業領域が不足したときは std::bad_alloc を送出します。
matrix(k,room) は次数の低下量が k 以下の変換行列用領域を返します。room は保存する変換の最大長で、保存しないときは 0 にします。返した行列の係数配列は HalfGcd の作業領域を参照するため、その寿命内だけ使います。
jump(a,n,b,m,k,out,need) は deg(a)=n > deg(b)=m、0 <= k <= n の入力で、互除法を deg(b) < n-k になるまでまとめます。零多項式の b は m=-1 で表します。入力係数は \([0,998244353)\)、最高次係数は非零とします。out は matrix(k,room) で確保します。need の bit 2*i+j は行列の \((i,j)\) 成分を求めるかを指定し、既定値は 15 です。指定しない成分の係数と長さは使用しません。
Matrix::progress は最初の多項式の次数低下量です。entry[i]・size[i] は係数配列とその長さ、at0[i] は定数項です。小さい jump は除算を繰り返さない互除法で求めます。このため行列全体の非零なスカラー倍は通常の互除法の行列と異なる場合があります。公開関数が最終的な GCD と Bézout 係数を正規化します。行列式が常に \(\pm1\) であるとは仮定しません。
native_inv_mod(a,n,b,m,out) は Mint 配列を受け取り、逆元が存在すれば空の out に追加して真を返します。b は非空で最高次係数が非零です。定数の法では空の結果を返します。native_gcd_matrix(a,n,b,m,need,sink) は n>m>0 の入力の jump を求め、行列が有効な間に sink(matrix) を呼びます。呼び出し側の入力は変更せず、追加の整列や padding を要求しません。
計算量: 最大係数数を \(n\) として、Half-GCD・GCD・拡張 GCD・法多項式に対する逆元は \(O(n\log^2 n)\) 時間、追加空間は \(O(n)\) です。
#ifndef CP_COMPONENTS_POLYNOMIAL_GCD_BLOCKED_HPP
#define CP_COMPONENTS_POLYNOMIAL_GCD_BLOCKED_HPP
#include "fps_division_blocked.hpp"
#include <limits>
#if defined(__linux__) && defined(__x86_64__) && defined(__GNUC__) && !defined(__clang__)
#pragma GCC push_options
_Pragma("GCC target(\"avx2\")")
#pragma GCC optimize("O3")
namespace fps_detail::blocked_poly {
namespace gcd_detail {
using namespace detail;
using division_detail::fold, division_detail::subtract, division_detail::montgomery_reduce, division_detail::kLongDivision;
class Stack {
public:
explicit Stack(Words memory) : base_(memory.data()), size_(memory.size()) {}
size_t mark() const { return top_; }
void release(size_t mark) { top_ = mark; }
Word* take(size_t n) {
const size_t need = Arena::footprint(n);
if (need > size_ - top_) throw std::bad_alloc();
Word* const p = base_ + top_;
top_ += need;
return p;
}
private:
Word* base_;
size_t size_, top_ = 0;
};
inline Word mod_mul(Word x, Word y) { return Word(std::uint64_t(x) * y % kP); }
inline Word mod_sub(Word x, Word y) { return x >= y ? x - y : x + kP - y; }
inline Word mod_negate(Word x) { return x ? kP - x : 0; }
inline Word mod_inverse(Word x) { return blocked_ntt::detail::power(x, kP - 2); }
inline void combine3(const Word* u, const Word* v, Word* out, std::ptrdiff_t from,
std::ptrdiff_t to, Word x, Word y, Word z) {
const Vec xs = broadcast(x), ys = broadcast(y), zs = broadcast(z);
for (std::ptrdiff_t j = from & ~std::ptrdiff_t(7); j < to; j += 8) {
const Vec a = load_unaligned(u + j), b = load_unaligned(v + j - 1), c = load_unaligned(v + j);
const Vec even = _mm256_add_epi64(_mm256_add_epi64(_mm256_mul_epu32(a, xs), _mm256_mul_epu32(b, ys)),
_mm256_mul_epu32(c, zs));
const Vec odd = _mm256_add_epi64(
_mm256_add_epi64(_mm256_mul_epu32(_mm256_srli_epi64(a, 32), xs), _mm256_mul_epu32(_mm256_srli_epi64(b, 32), ys)),
_mm256_mul_epu32(_mm256_srli_epi64(c, 32), zs));
store_unaligned(out + j, reduce(montgomery_reduce(even, odd), kP));
}
}
[[gnu::always_inline]] inline void fill_pair(const Word* roots, const Word* x0, const Word* x1,
size_t g, Window (&window)[2][4]) {
const size_t at = 32 * g;
const Vec a0[4] = {load(x0 + at), load(x0 + at + 8), load(x0 + at + 16), load(x0 + at + 24)};
const Vec a1[4] = {load(x1 + at), load(x1 + at + 8), load(x1 + at + 16), load(x1 + at + 24)};
fill_windows(window[0], a0, roots, g);
fill_windows(window[1], a1, roots, g);
}
template <size_t K>
inline void products(const Word* roots, const Word* x0, const Word* x1,
const Word* const (&y_in)[K][2], Word* const (&out_in)[K], size_t n) {
const Word* y[K][2];
Word* out[K];
for (size_t k = 0; k < K; ++k) y[k][0] = y_in[k][0], y[k][1] = y_in[k][1], out[k] = out_in[k];
const Factor undo_montgomery(kR);
Window window[2][2][4];
fill_pair(roots, x0, x1, 0, window[0]);
for (size_t g = 0; g < n / 32; ++g) {
if (g + 1 < n / 32) fill_pair(roots, x0, x1, g + 1, window[(g + 1) & 1]);
const Window(&w)[2][4] = window[g & 1];
const size_t at = 32 * g;
#pragma GCC unroll 4
for (size_t t = 0; t < 4; ++t)
for (size_t k = 0; k < K; ++k) {
const Vec s = low(add(leaf_product(w[0][t], y[k][0] + at + 8 * t), leaf_product(w[1][t], y[k][1] + at + 8 * t)));
store(out[k] + at + 8 * t, reduce(times(s, undo_montgomery), kP));
}
}
}
struct Matrix {
Word* entry[4];
size_t size[4];
Word at0[4];
Word* transform[4];
size_t length[4];
size_t room, progress;
};
class HalfGcd {
public:
static constexpr std::ptrdiff_t kDirect = 64;
static int log_length(size_t n_max) {
return std::max(Transform::kMinLog, int(std::bit_width(std::max<size_t>(n_max, 2) - 1)));
}
static size_t scratch_words(size_t n_max) { return 48 * (size_t(1) << log_length(n_max)) + 4096; }
static size_t words(size_t n_max) {
return Transform::words(log_length(n_max)) + Arena::footprint(scratch_words(n_max));
}
HalfGcd(Arena& arena, size_t n_max, std::ptrdiff_t direct = kDirect)
: transform_(arena, log_length(n_max)), stack_(arena.take(scratch_words(n_max))), direct_(direct) {}
Matrix matrix(size_t k, size_t room) {
Matrix r;
for (int i = 0; i < 4; ++i) {
r.entry[i] = take(k + 1).data();
r.transform[i] = room ? take(room).data() : nullptr;
r.length[i] = 0;
}
r.room = room;
return r;
}
void jump(const Word* a, std::ptrdiff_t n, const Word* b, std::ptrdiff_t m, std::ptrdiff_t k,
Matrix& out, unsigned need = 0xF) {
const std::ptrdiff_t target = n - k;
if (m < target) return identity(out);
if (n > 2 * k) {
const std::ptrdiff_t s = n - 2 * k;
return jump(a + s, n - s, b + s, m - s, k, out, need);
}
if (k <= direct_) return euclid(a, n, b, m, k, out);
const size_t mark = stack_.mark(), room = std::max<size_t>(64, std::bit_ceil(size_t(k)));
const std::ptrdiff_t k1 = first_jump(k), s1 = std::max<std::ptrdiff_t>(0, n - 2 * k1);
Matrix r1 = matrix(size_t(k1), room);
jump(a + s1, n - s1, b + s1, m - s1, k1, r1);
std::ptrdiff_t c_deg = 0;
const Word *c = nullptr, *d = nullptr;
if (r1.progress == 0) {
r1 = matrix(size_t(n - m), room);
Word* const rest = take(size_t(n + 1)).data();
divide_step(a, n, b, m, r1, rest);
c = b, d = rest, c_deg = m;
} else {
c_deg = n - std::ptrdiff_t(r1.progress);
const std::ptrdiff_t s2 = std::max<std::ptrdiff_t>(0, 2 * target - c_deg);
Word *const cs = take(size_t(c_deg + 1)).data(), *const ds = take(size_t(c_deg + 1)).data();
apply(a, n, b, m, r1, c_deg, s2, cs, ds);
c = cs, d = ds;
}
std::ptrdiff_t d_deg = c_deg - 1;
while (d_deg >= target && d[d_deg] == 0) --d_deg;
if (d_deg < target) {
copy(r1, out);
return stack_.release(mark);
}
const std::ptrdiff_t k2 = c_deg - target, s2 = std::max<std::ptrdiff_t>(0, c_deg - 2 * k2);
Matrix r2 = matrix(size_t(k2), room);
jump(c + s2, c_deg - s2, d + s2, d_deg - s2, k2, r2, (need & 3 ? 3u : 0u) | (need & 12 ? 12u : 0u));
combine(r2, r1, out, need);
stack_.release(mark);
}
private:
static std::ptrdiff_t first_jump(std::ptrdiff_t k) {
const size_t u = size_t(k);
return std::ptrdiff_t(std::has_single_bit(u) ? u / 2 : std::bit_floor(u));
}
Words take(size_t n) { return {stack_.take(n), n}; }
Word* take_padded(size_t n) {
Word* const p = stack_.take(n + 16);
std::fill_n(p, n + 16, 0);
return p + 8;
}
static void identity(Matrix& out) {
out.entry[0][0] = out.entry[3][0] = 1;
out.size[0] = out.size[3] = 1;
out.size[1] = out.size[2] = 0;
out.at0[0] = out.at0[3] = 1, out.at0[1] = out.at0[2] = 0;
for (auto& l : out.length) l = 0;
out.progress = 0;
}
static void copy(const Matrix& from, Matrix& to) {
for (int i = 0; i < 4; ++i) {
std::copy_n(from.entry[i], from.size[i], to.entry[i]);
to.size[i] = from.size[i];
to.at0[i] = from.at0[i];
to.length[i] = 0;
}
to.progress = from.progress;
}
void euclid(const Word* a, std::ptrdiff_t n, const Word* b, std::ptrdiff_t m, std::ptrdiff_t k,
Matrix& out) {
const size_t mark = stack_.mark();
const std::ptrdiff_t target = n - k;
std::ptrdiff_t floor = n - 2 * k, lo = std::max<std::ptrdiff_t>(0, floor);
Word* p[3];
for (auto& x : p) x = take_padded(size_t(n + 1));
std::copy(a + lo, a + n + 1, p[0] + lo);
std::copy(b + lo, b + m + 1, p[1] + lo);
const size_t cap = size_t(k) + 8;
Word* row[3][2];
size_t row_size[3][2] = {{1, 0}, {0, 1}, {0, 0}};
for (auto& r : row)
for (auto& e : r) {
e = take_padded(cap);
std::fill_n(e, cap, 0);
}
row[0][0][0] = 1, row[1][1][0] = 1;
Word *c = p[0], *d = p[1], *e = p[2];
std::ptrdiff_t c_deg = n, d_deg = m;
while (d_deg >= target) {
const std::ptrdiff_t delta = c_deg - d_deg;
const Word alpha = d[d_deg];
floor += delta;
lo = std::max<std::ptrdiff_t>(0, floor);
if (delta == 1) {
const Word beta = c[c_deg];
const Word gamma = mod_sub(mod_mul(alpha, c[c_deg - 1]), mod_mul(beta, d[d_deg - 1]));
const Word alpha_r = mod_mul(alpha, kR);
const Word x = mod_mul(alpha, alpha_r), y = kP - mod_mul(beta, alpha_r), z = mod_negate(mod_mul(gamma, kR));
combine3(c, d, e, lo, d_deg, x, y, z);
for (int i = 0; i < 2; ++i) {
const size_t sc = row_size[0][i], sd = row_size[1][i];
const size_t se = std::max(sc, sd ? sd + 1 : 0);
combine3(row[0][i], row[1][i], row[2][i], 0, std::ptrdiff_t(se), x, y, z);
if (row_size[2][i] > se) std::fill(row[2][i] + se, row[2][i] + row_size[2][i], 0);
row_size[2][i] = se;
}
} else {
pseudo_divide(c, c_deg, d, d_deg, e, lo, alpha, row, row_size);
}
Word* const old = c;
c = d, d = e, e = old;
for (int i = 0; i < 2; ++i) {
std::swap(row[0][i], row[1][i]);
std::swap(row[1][i], row[2][i]);
std::swap(row_size[0][i], row_size[1][i]);
std::swap(row_size[1][i], row_size[2][i]);
}
c_deg = d_deg;
d_deg = c_deg - 1;
while (d_deg >= target && d[d_deg] == 0) --d_deg;
}
for (int i = 0; i < 2; ++i)
for (int j = 0; j < 2; ++j) {
std::copy_n(row[i][j], row_size[i][j], out.entry[2 * i + j]);
out.size[2 * i + j] = row_size[i][j];
out.at0[2 * i + j] = row_size[i][j] ? row[i][j][0] : 0;
out.length[2 * i + j] = 0;
}
out.progress = size_t(n - c_deg);
stack_.release(mark);
}
static void pseudo_divide(const Word* c, std::ptrdiff_t c_deg, const Word* d, std::ptrdiff_t d_deg,
Word* e, std::ptrdiff_t lo, Word alpha, Word* (&row)[3][2],
size_t (&row_size)[3][2]) {
std::copy(c + lo, c + c_deg + 1, e + lo);
for (int i = 0; i < 2; ++i) {
std::copy_n(row[0][i], row_size[0][i], row[2][i]);
if (row_size[2][i] > row_size[0][i]) std::fill(row[2][i] + row_size[0][i], row[2][i] + row_size[2][i], 0);
row_size[2][i] = row_size[0][i];
}
for (std::ptrdiff_t t = c_deg - d_deg; t >= 0; --t) {
const Word minus_tau = mod_negate(e[d_deg + t]);
for (std::ptrdiff_t j = lo; j <= d_deg + t; ++j) {
const Word dj = j - t >= 0 ? d[j - t] : 0;
e[j] = Word((std::uint64_t(alpha) * e[j] + std::uint64_t(minus_tau) * dj) % kP);
}
for (int i = 0; i < 2; ++i) {
Word* const re = row[2][i];
const Word* const rd = row[1][i];
const size_t sd = row_size[1][i], shift = size_t(t);
const size_t se = std::max(row_size[2][i], sd ? sd + shift : 0);
for (size_t j = 0; j < se; ++j) {
const Word dj = j >= shift ? rd[j - shift] : 0;
re[j] = Word((std::uint64_t(alpha) * re[j] + std::uint64_t(minus_tau) * dj) % kP);
}
row_size[2][i] = se;
}
}
}
void divide_step(const Word* a, std::ptrdiff_t n, const Word* b, std::ptrdiff_t m, Matrix& r,
Word* rest) {
const size_t delta = size_t(n - m);
Word* q = r.entry[3];
if ((delta + 1) * size_t(m + 1) <= kLongDivision) {
std::copy(a, a + n + 1, rest);
const Word lead = mod_inverse(b[m]);
for (size_t i = delta + 1; i-- > 0;) {
const Word qi = mod_mul(rest[size_t(m) + i], lead);
q[i] = qi;
for (std::ptrdiff_t t = 0; t <= m; ++t)
rest[i + size_t(t)] = mod_sub(rest[i + size_t(t)], mod_mul(qi, b[t]));
}
} else {
const ConstWords f(a, size_t(n + 1)), g(b, size_t(m + 1));
Arena arena(divide_words(f.size(), g.size()));
divide(arena, f, g, Words(q, delta + 1), Words(rest, size_t(m)));
std::fill(rest + m, rest + n + 1, 0);
}
for (size_t i = 0; i <= delta; ++i) q[i] = mod_negate(q[i]);
r.entry[1][0] = r.entry[2][0] = 1;
r.size[0] = 0, r.size[1] = r.size[2] = 1, r.size[3] = delta + 1;
r.at0[0] = 0, r.at0[1] = r.at0[2] = 1, r.at0[3] = q[0];
for (auto& l : r.length) l = 0;
r.progress = delta;
}
void transform_of(const Word* c, size_t size, Words out) const {
const size_t len = out.size();
if (size <= len) return transform_.forward(ConstWords(c, size), 0, out);
fold(ConstWords(c, size), out);
transform_.forward(out);
}
void extend(Matrix& r, int i, size_t len) const {
if (r.length[i] == len) return;
const Words t(r.transform[i], len);
const size_t half = len / 2, size = r.size[i];
if (r.length[i] == half && size <= half + 1) {
const Words upper = t.subspan(half);
if (size <= half) {
transform_.forward_upper(ConstWords(r.entry[i], size), 0, upper);
} else {
std::copy_n(r.entry[i], half, upper.data());
upper[0] = mod_sub(upper[0], r.entry[i][half]);
transform_.forward_upper(upper, 0, upper);
}
} else {
transform_of(r.entry[i], size, t);
}
r.length[i] = len;
}
void apply(const Word* a, std::ptrdiff_t n, const Word* b, std::ptrdiff_t m, Matrix& r1,
std::ptrdiff_t c_deg, std::ptrdiff_t s2, Word* c, Word* d) {
const size_t mark = stack_.mark();
const size_t len = std::max<size_t>(64, std::bit_ceil(size_t(n + 1) / 2));
for (int i = 0; i < 4; ++i) extend(r1, i, len);
const Words ta = take(len), tb = take(len);
transform_of(a, size_t(n + 1), ta);
transform_of(b, size_t(m + 1), tb);
const Words pc = take(len), pd = take(len);
apply_products(r1, ta, tb, pc, pd);
const std::ptrdiff_t l = std::ptrdiff_t(len), mixed = c_deg - l;
if (mixed < 0) {
std::copy(pc.begin() + s2, pc.begin() + c_deg + 1, c + s2);
std::copy(pd.begin() + s2, pd.begin() + c_deg + 1, d + s2);
return stack_.release(mark);
}
const size_t low = size_t(mixed + 1);
const Words lc = take(len), ld = take(len);
const size_t short_len = std::max<size_t>(64, std::bit_ceil(2 * low - 1));
if (short_len < len) {
Matrix r = r1;
for (int i = 0; i < 4; ++i) {
r.transform[i] = take(short_len).data();
transform_.forward(ConstWords(r1.entry[i], std::min(low, r1.size[i])), 0,
Words(r.transform[i], short_len));
}
const Words sa = ta.first(short_len), sb = tb.first(short_len);
transform_.forward(ConstWords(a, low), 0, sa);
transform_.forward(ConstWords(b, std::min(low, size_t(m + 1))), 0, sb);
apply_products(r, sa, sb, lc.first(short_len), ld.first(short_len));
} else {
transform_.forward(ConstWords(a, low), 0, ta);
transform_.forward(ConstWords(b, std::min(low, size_t(m + 1))), 0, tb);
apply_products(r1, ta, tb, lc, ld);
lc[0] = Word((std::uint64_t(r1.at0[0]) * a[0] + std::uint64_t(r1.at0[1]) * b[0]) % kP);
ld[0] = Word((std::uint64_t(r1.at0[2]) * a[0] + std::uint64_t(r1.at0[3]) * b[0]) % kP);
}
const std::ptrdiff_t first = std::min(s2, mixed + 1);
std::copy(lc.begin() + first, lc.begin() + mixed + 1, c + first);
std::copy(ld.begin() + first, ld.begin() + mixed + 1, d + first);
const std::ptrdiff_t from = std::max(s2, mixed + 1);
std::copy(pc.begin() + from, pc.begin() + l, c + from);
std::copy(pd.begin() + from, pd.begin() + l, d + from);
subtract(pc.data(), lc.data(), c + l, size_t(mixed + 1));
subtract(pd.data(), ld.data(), d + l, size_t(mixed + 1));
stack_.release(mark);
}
void apply_products(const Matrix& r1, ConstWords ta, ConstWords tb,
Words c, Words d) const {
const size_t len = c.size();
const auto t = [&](int i) { return ConstWords(r1.transform[i], len); };
const Transform::Pair pc[2] = {{t(0), ta}, {t(1), tb}}, pd[2] = {{t(2), ta}, {t(3), tb}};
transform_.inverse_product_sum(pc, c);
transform_.inverse_product_sum(pd, d);
}
void combine(Matrix& r2, Matrix& r1, Matrix& out, unsigned need) {
const size_t mark = stack_.mark();
const size_t p = r2.progress + r1.progress, len = std::max<size_t>(64, std::bit_ceil(p));
for (int i = 0; i < 2; ++i)
if (need & (3u << 2 * i)) extend(r2, 2 * i, len), extend(r2, 2 * i + 1, len);
for (int j = 0; j < 2; ++j)
if (need & (5u << j)) extend(r1, j, len), extend(r1, 2 + j, len);
const Words buffer = take(len);
const bool keep = out.room >= len;
Word* t[4];
for (int i = 0; i < 4; ++i) t[i] = keep ? out.transform[i] : take(len).data();
for (int i = 0; i < 2; ++i) {
const Word *x0 = r2.transform[2 * i], *x1 = r2.transform[2 * i + 1];
const Word* const y[2][2] = {{r1.transform[0], r1.transform[2]}, {r1.transform[1], r1.transform[3]}};
const unsigned row = need >> 2 * i & 3;
if (row == 3) {
Word* const to[2] = {t[2 * i], t[2 * i + 1]};
products<2>(transform_.roots(), x0, x1, y, to, len);
} else if (row) {
const int j = row == 1 ? 0 : 1;
const Word* const yj[1][2] = {{y[j][0], y[j][1]}};
Word* const to[1] = {t[2 * i + j]};
products<1>(transform_.roots(), x0, x1, yj, to, len);
}
}
for (int i = 0; i < 2; ++i)
for (int j = 0; j < 2; ++j) {
const int ij = 2 * i + j;
out.length[ij] = 0;
const Word x0 = Word(
(std::uint64_t(r2.at0[2 * i]) * r1.at0[j] + std::uint64_t(r2.at0[2 * i + 1]) * r1.at0[2 + j]) % kP);
out.at0[ij] = x0;
if (!(need >> ij & 1)) continue;
if (keep) out.length[ij] = len;
transform_.inverse(ConstWords(t[ij], len), buffer);
Word* const e = out.entry[ij];
std::copy_n(buffer.data(), std::min(p + 1, len), e);
if (p == len) e[len] = mod_sub(e[0], x0), e[0] = x0;
size_t size = p + 1;
while (size > 0 && e[size - 1] == 0) --size;
out.size[ij] = size;
}
out.progress = p;
stack_.release(mark);
}
Transform transform_;
Stack stack_;
std::ptrdiff_t direct_;
};
}
inline size_t inverse_mod_words(size_t n, size_t m) {
const size_t d = m - 1;
return divide_words(n, m) + Arena::footprint(d) + Arena::footprint(quotient_size(n, m)) +
gcd_detail::HalfGcd::words(std::max<size_t>(d, 1));
}
inline std::ptrdiff_t inverse_mod(Arena& arena, ConstWords f, ConstWords g,
Words h) {
using namespace gcd_detail;
const std::ptrdiff_t d = std::ptrdiff_t(g.size()) - 1;
if (d == 0) return 0;
const Words b = arena.take(size_t(d));
const size_t rs = remainder_size(f.size(), g.size());
divide(arena, f, g, arena.take(quotient_size(f.size(), g.size())), b.first(rs));
std::ptrdiff_t m = std::ptrdiff_t(rs) - 1;
while (m >= 0 && b[size_t(m)] == 0) --m;
if (m < 0) return -1;
HalfGcd gcd(arena, size_t(d));
Matrix r = gcd.matrix(size_t(d), 0);
gcd.jump(g.data(), d, b.data(), m, d, r, 0x2);
if (std::ptrdiff_t(r.progress) != d) return -1;
const Word c = Word((std::uint64_t(r.at0[0]) * g[0] + std::uint64_t(r.at0[1]) * b[0]) % kP);
const Word scale = mod_inverse(c);
size_t t = r.size[1];
while (t > 0 && r.entry[1][t - 1] == 0) --t;
for (size_t i = 0; i < t; ++i) h[i] = mod_mul(r.entry[1][i], scale);
return std::ptrdiff_t(t);
}
}
namespace fps_detail::blocked_poly {
inline bool native_inv_mod(const mint* a, size_t n, const mint* b, size_t m, std::vector<mint>& out) {
Arena arena(inverse_mod_words(n, m) + Arena::footprint(n) + Arena::footprint(m) + Arena::footprint(m));
auto f = arena.take(n), g = arena.take(m), h = arena.take(m);
for (size_t i = 0; i < n; ++i) f[i] = std::min(a[i].v, a[i].v - mint::mod());
for (size_t i = 0; i < m; ++i) g[i] = std::min(b[i].v, b[i].v - mint::mod());
const auto size = inverse_mod(arena, f, g, h);
if (size < 0) return false;
prepare_fps_output(out, size_t(size));
division_detail::append_words(out, h.data(), size_t(size), 932051910);
return true;
}
template<class Sink>
inline void native_gcd_matrix(const mint* a, size_t n, const mint* b, size_t m, unsigned need, Sink sink) {
Arena arena(gcd_detail::HalfGcd::words(n - 1) + Arena::footprint(n) + Arena::footprint(m));
auto f = arena.take(n), g = arena.take(m);
for (size_t i = 0; i < n; ++i) f[i] = std::min(a[i].v, a[i].v - mint::mod());
for (size_t i = 0; i < m; ++i) g[i] = std::min(b[i].v, b[i].v - mint::mod());
gcd_detail::HalfGcd gcd(arena, n - 1);
auto matrix = gcd.matrix(n - 1, 0);
gcd.jump(f.data(), std::ptrdiff_t(n) - 1, g.data(), std::ptrdiff_t(m) - 1, std::ptrdiff_t(n) - 1, matrix, need);
sink(matrix);
}
}
#pragma GCC pop_options
#endif
#endif
#line 1 "components/polynomial_gcd_blocked.hpp"
#line 1 "components/fps_division_blocked.hpp"
#line 1 "components/fps_blocked.hpp"
#line 1 "components/fps_workspace.hpp"
#include <cstddef>
#include <cstdint>
#include <limits>
#include <memory>
#include <new>
#if defined(__linux__)
#include <sys/mman.h>
#endif
namespace fps_detail {
struct FpsWordsDelete {
void *region = nullptr;
std::size_t bytes = 0;
void operator()(unsigned int *p) const {
#if defined(__linux__)
if (region) {
munmap(region, bytes);
return;
}
#endif
::operator delete[](p, std::align_val_t(64));
}
};
using FpsWords = std::unique_ptr<unsigned int[], FpsWordsDelete>;
inline FpsWords allocate_fps_words(std::size_t count) {
#if defined(__linux__) && defined(MADV_HUGEPAGE)
constexpr std::size_t page = 1 << 21;
if (count >= page / sizeof(unsigned int) &&
count <= (std::numeric_limits<std::size_t>::max() - 2 * page) / sizeof(unsigned int)) {
const std::size_t bytes = (count * sizeof(unsigned int) + page - 1) / page * page;
void *region = mmap(nullptr, bytes + page, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (region != MAP_FAILED) {
const auto address = (reinterpret_cast<std::uintptr_t>(region) + page - 1) & ~(page - 1);
auto *p = ::new (reinterpret_cast<void *>(address)) unsigned int[count];
madvise(p, bytes, MADV_HUGEPAGE);
return FpsWords(p, FpsWordsDelete{region, bytes + page});
}
}
#endif
return FpsWords(new (std::align_val_t(64)) unsigned int[count]);
}
}
#line 1 "components/fps_output.hpp"
#line 1 "components/mint_arithmetic.hpp"
#line 1 "math/mint.hpp"
#include <cassert>
#include <istream>
#include <ostream>
#include <type_traits>
#if defined(__GNUC__) && !defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
#pragma GCC push_options
#pragma GCC optimize("O3")
#endif
namespace mint_detail {
template <class I>
constexpr bool mint_integral = std::is_integral_v<I>
#if defined(__SIZEOF_INT128__)
|| std::is_same_v<I, __int128> || std::is_same_v<I, unsigned __int128>
#endif
;
struct [[gnu::warn_unused]] Mint {
using mint = Mint;
unsigned v = 0;
[[nodiscard]] static constexpr unsigned mod() { return 998244353; }
[[nodiscard, gnu::always_inline]] static unsigned reduce(unsigned long long x) {
return (x + (unsigned long long)(unsigned(x) * 998244351u) * mod()) >> 32;
}
[[gnu::always_inline]] Mint() = default;
template <class I, std::enable_if_t<mint_integral<I>, int> = 0>
Mint(I x) {
long long r = x % (long long)mod();
if (r < 0) r += mod();
v = reduce((unsigned long long)r * 932051910);
}
[[nodiscard, gnu::always_inline]] static Mint raw(unsigned x) {
Mint a;
a.v = reduce((unsigned long long)x * 932051910);
return a;
}
[[nodiscard, gnu::always_inline]] static Mint word(unsigned x) {
Mint a;
a.v = x;
return a;
}
[[nodiscard, gnu::always_inline]] unsigned val() const {
unsigned r = reduce(v);
return r < mod() ? r : r - mod();
}
friend std::istream &operator>>(std::istream &is, Mint &x) {
long long value;
if (is >> value) x = Mint(value);
return is;
}
friend std::ostream &operator<<(std::ostream &os, const Mint &x) {
return os << x.val();
}
[[gnu::always_inline]] Mint &operator+=(Mint b) {
v += b.v;
if (v >= 2 * mod()) v -= 2 * mod();
return *this;
}
[[gnu::always_inline]] Mint &operator-=(Mint b) {
v -= b.v;
if (v >> 31) v += 2 * mod();
return *this;
}
[[gnu::always_inline]] Mint &operator*=(Mint b) {
v = reduce((unsigned long long)v * b.v);
return *this;
}
Mint &operator/=(Mint b) { return *this *= b.inv(); }
Mint &operator++() { return *this += Mint(1); }
Mint &operator--() { return *this -= Mint(1); }
Mint operator++(int) {
Mint a = *this;
++*this;
return a;
}
Mint operator--(int) {
Mint a = *this;
--*this;
return a;
}
[[nodiscard, gnu::always_inline]] Mint operator+() const { return *this; }
[[nodiscard, gnu::always_inline]] Mint operator-() const { return word(v ? 2 * mod() - v : 0); }
[[nodiscard, gnu::always_inline]] Mint operator+(Mint b) const { return Mint(*this) += b; }
[[nodiscard, gnu::always_inline]] Mint operator-(Mint b) const { return Mint(*this) -= b; }
[[nodiscard, gnu::always_inline]] Mint operator*(Mint b) const { return Mint(*this) *= b; }
[[nodiscard]] Mint operator/(Mint b) const { return Mint(*this) /= b; }
[[nodiscard, gnu::always_inline]] bool operator==(Mint b) const { return (v < mod() ? v : v - mod()) == (b.v < mod() ? b.v : b.v - mod()); }
[[nodiscard, gnu::always_inline]] bool operator!=(Mint b) const { return !(*this == b); }
template <class I, std::enable_if_t<mint_integral<I>, int> = 0>
[[nodiscard]] friend Mint operator+(I a, Mint b) { return Mint(a) + b; }
template <class I, std::enable_if_t<mint_integral<I>, int> = 0>
[[nodiscard]] friend Mint operator-(I a, Mint b) { return Mint(a) - b; }
template <class I, std::enable_if_t<mint_integral<I>, int> = 0>
[[nodiscard]] friend Mint operator*(I a, Mint b) { return Mint(a) * b; }
template <class I, std::enable_if_t<mint_integral<I>, int> = 0>
[[nodiscard]] friend Mint operator/(I a, Mint b) { return Mint(a) / b; }
template <class I, std::enable_if_t<mint_integral<I>, int> = 0>
[[nodiscard]] friend bool operator==(I a, Mint b) { return Mint(a) == b; }
template <class I, std::enable_if_t<mint_integral<I>, int> = 0>
[[nodiscard]] friend bool operator!=(I a, Mint b) { return Mint(a) != b; }
[[nodiscard]] Mint pow(long long n) const {
assert(n >= 0);
if (!n) return Mint(1);
Mint r(1), a = *this;
for (; n > 1; n >>= 1, a *= a)
if (n & 1) r *= a;
return r * a;
}
[[nodiscard, gnu::noinline]] Mint inv() const {
assert(*this != Mint(0));
Mint a2 = *this * *this, a3 = a2 * *this;
Mint a7 = a3 * a3 * *this;
Mint a31 = a7;
for (int i = 0; i < 2; ++i) a31 *= a31;
a31 *= a3;
Mint r = a7;
for (int i = 0; i < 3; ++i) r *= r;
r *= a3;
r *= r;
for (int j = 0; j < 4; ++j) {
for (int i = 0; i < 5; ++i) r *= r;
r *= a31;
}
for (int i = 0; i < 3; ++i) r *= r;
return r * a7;
}
};
} // namespace mint_detail
using mint = mint_detail::Mint;
using fps_mint = mint;
#if defined(__GNUC__) && !defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
#pragma GCC pop_options
#endif
#line 5 "components/mint_arithmetic.hpp"
#if defined(__x86_64__) || defined(__i386__)
#include <immintrin.h>
#endif
#if defined(__GNUC__) && !defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
#pragma GCC push_options
#pragma GCC optimize("O3")
#endif
namespace mint_detail {
struct Ops {
using T = mint;
typedef unsigned int U __attribute__((may_alias));
static constexpr U mod = 998244353, twice = 2 * mod, one = 301989884;
static U reduce(unsigned long long x) { return T::reduce(x); }
static U mul(U a, U b) { return reduce((unsigned long long)a * b); }
static U add(U a, U b) {
U c = a + b;
return c < twice ? c : c - twice;
}
static U sub(U a, U b) {
U c = a - b;
return c < twice ? c : c + twice;
}
static U in(U a) { return mul(a, 932051910); }
static U out(U a) {
a = reduce(a);
return a < mod ? a : a - mod;
}
static U quotient(U r) { return ((unsigned long long)r << 32) / mod; }
static U fixed_mul(U a, U r, U q) {
return a * r - U(((unsigned long long)a * q) >> 32) * mod;
}
#if defined(__x86_64__) || defined(__i386__)
static bool vectorized() {
#if defined(__AVX2__)
return true;
#else
static const bool available = __builtin_cpu_supports("avx2");
return available;
#endif
}
using I8 = __m256i;
__attribute__((target("avx2"))) static I8 load8(const U *a) {
return _mm256_loadu_si256(reinterpret_cast<const I8 *>(a));
}
__attribute__((target("avx2"))) static void store8(U *a, I8 x) {
_mm256_storeu_si256(reinterpret_cast<I8 *>(a), x);
}
__attribute__((target("avx2"), always_inline)) static inline I8 reduce8(I8 x, I8 y) {
I8 r = _mm256_set1_epi32(998244351), p = _mm256_set1_epi32(mod);
x = _mm256_add_epi64(x, _mm256_mul_epu32(_mm256_mul_epu32(x, r), p));
y = _mm256_add_epi64(y, _mm256_mul_epu32(_mm256_mul_epu32(y, r), p));
return _mm256_blend_epi32(_mm256_srli_epi64(x, 32), y, 0xaa);
}
__attribute__((target("avx2"))) static I8 mul8(I8 a, I8 b) {
return reduce8(_mm256_mul_epu32(a, b), _mm256_mul_epu32(_mm256_srli_epi64(a, 32), _mm256_srli_epi64(b, 32)));
}
__attribute__((target("avx2"))) static I8 fixed_mul8(I8 a, I8 r, I8 q) {
I8 x = _mm256_srli_epi64(_mm256_mul_epu32(a, q), 32);
I8 y = _mm256_mul_epu32(_mm256_srli_epi64(a, 32), q);
I8 t = _mm256_blend_epi32(x, y, 0xaa);
return _mm256_sub_epi32(_mm256_mullo_epi32(a, r),
_mm256_mullo_epi32(t, _mm256_set1_epi32(mod)));
}
__attribute__((target("avx2"))) static I8 fixed_even8(I8 a, I8 r, I8 q) {
I8 t = _mm256_srli_epi64(_mm256_mul_epu32(a, q), 32);
return _mm256_sub_epi64(_mm256_mul_epu32(a, r),
_mm256_mul_epu32(t, _mm256_set1_epi32(mod)));
}
__attribute__((target("avx2"))) static I8 add8(I8 a, I8 b) {
I8 x = _mm256_add_epi32(a, b);
return _mm256_min_epu32(x, _mm256_sub_epi32(x, _mm256_set1_epi32(twice)));
}
__attribute__((target("avx2"))) static I8 sub8(I8 a, I8 b) {
I8 x = _mm256_sub_epi32(a, b);
return _mm256_min_epu32(x, _mm256_add_epi32(x, _mm256_set1_epi32(twice)));
}
#endif
};
}
#if defined(__GNUC__) && !defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
#pragma GCC pop_options
#endif
#line 5 "components/fps_output.hpp"
#include <array>
#include <cstring>
#line 8 "components/fps_output.hpp"
#include <stdexcept>
#line 10 "components/fps_output.hpp"
#include <vector>
#if defined(__linux__) && defined(__x86_64__) && defined(__GNUC__) && !defined(__clang__)
#include <immintrin.h>
#include <sys/mman.h>
#pragma GCC push_options
_Pragma("GCC target(\"avx2\")")
#pragma GCC optimize("O3")
namespace fps_detail {
inline void prepare_fps_output(std::vector<mint> &out, std::size_t n) {
if (n <= out.capacity()) return;
out.reserve(n);
if (n - out.size() < (1u << 17)) return;
const auto first = reinterpret_cast<std::uintptr_t>(out.data() + out.size());
const auto end = reinterpret_cast<std::uintptr_t>(out.data() + n);
[[maybe_unused]] const auto advise = [first, end](std::uintptr_t alignment, int advice) {
const auto lo = (first + alignment - 1) & ~(alignment - 1), hi = end & ~(alignment - 1);
if (lo < hi) static_cast<void>(madvise(reinterpret_cast<void *>(lo), hi - lo, advice));
};
#if defined(MADV_HUGEPAGE)
advise(2u * 1024 * 1024, MADV_HUGEPAGE);
#endif
#if defined(MADV_POPULATE_WRITE)
advise(4096, MADV_POPULATE_WRITE);
#endif
}
template <bool montgomery>
inline void append_fps_words(std::vector<mint> &out, const unsigned *in, std::size_t n) {
static_assert(sizeof(mint) == sizeof(unsigned) && std::is_trivially_copyable_v<mint>);
if (n == 0) return;
if (n > out.max_size() - out.size()) throw std::length_error("fps output");
prepare_fps_output(out, out.size() + n);
std::array<mint, 64> batch;
std::size_t i = 0;
for (; i + 64 <= n; i += 64) {
if constexpr (montgomery) std::memcpy(static_cast<void *>(batch.data()), in + i, sizeof(batch));
else {
#pragma GCC unroll 8
for (int j = 0; j < 64; j += 8) {
const auto x = mint_detail::Ops::mul8(_mm256_loadu_si256(reinterpret_cast<const __m256i *>(in + i + j)), _mm256_set1_epi32(932051910u));
std::memcpy(static_cast<void *>(batch.data() + j), &x, sizeof(x));
}
}
out.insert(out.end(), batch.begin(), batch.end());
}
for (; i < n; ++i) {
if constexpr (montgomery) out.emplace_back(mint::word(in[i]));
else out.emplace_back(mint::raw(in[i]));
}
}
}
#pragma GCC pop_options
#endif
#line 6 "components/fps_blocked.hpp"
#include <algorithm>
#line 8 "components/fps_blocked.hpp"
#include <bit>
#line 11 "components/fps_blocked.hpp"
#include <cstdlib>
#line 13 "components/fps_blocked.hpp"
#include <span>
#line 16 "components/fps_blocked.hpp"
#if defined(__linux__) && defined(__x86_64__) && defined(__GNUC__) && !defined(__clang__)
#include <immintrin.h>
#pragma GCC push_options
_Pragma("GCC target(\"avx2\")")
#pragma GCC optimize("O3")
namespace fps_detail {
using Word = std::uint32_t;
using Words = std::span<Word>;
using ConstWords = std::span<const Word>;
using std::size_t;
namespace blocked_ntt::kernels {
using Vec = __m256i;
inline constexpr Word kP = 998244353, k2P = 2 * kP, kNI = 998244351;
}
namespace blocked_ntt {
inline constexpr Word kModulus = kernels::kP;
inline constexpr int kMaxLog = 25;
namespace detail {
using kernels::Vec, kernels::kP;
constexpr Word multiply_mod(Word x, Word y) {
return Word(std::uint64_t(x) * y % kP);
}
constexpr Word power(Word x, Word e) {
Word r = 1;
for (; e; e >>= 1, x = multiply_mod(x, x))
if (e & 1) r = multiply_mod(r, x);
return r;
}
constexpr Word quotient(Word w) { return Word((std::uint64_t(w) << 32) / kP); }
inline constexpr Word kR = Word((std::uint64_t(1) << 32) % kP);
inline constexpr int kRootCount = kMaxLog - 4;
inline constexpr auto kRoots = [] {
std::array<std::array<Word, kRootCount>, 2> t{};
for (int j = 0; j < kRootCount; ++j) {
t[0][j] = power(3, (kP - 1) >> (j + 2));
t[1][j] = power(t[0][j], kP - 2);
}
return t;
}();
inline Vec broadcast(Word x) { return _mm256_set1_epi32(int(x)); }
inline Vec add(Vec x, Vec y) { return _mm256_add_epi32(x, y); }
inline Vec diff(Vec x, Vec y) { return _mm256_sub_epi32(_mm256_add_epi32(x, broadcast(2 * kP)), y); }
inline Vec reduce(Vec x, Word bound) { return _mm256_min_epu32(x, _mm256_sub_epi32(x, broadcast(bound))); }
struct Factor {
Vec w, q;
explicit Factor(Word value) : w(broadcast(value)), q(broadcast(quotient(value))) {}
Factor(Word value, Word value_quotient) : w(broadcast(value)), q(broadcast(value_quotient)) {}
};
inline Vec multiply(Vec x, const Factor& f) { return mint_detail::Ops::fixed_mul8(x, f.w, f.q); }
inline size_t slot(size_t k) { return (k >> 3 << 4) | (k & 7); }
inline constexpr size_t table_words(int lg) { return (size_t(1) << lg) / 8 + 16; }
inline void build_table(Word* t, size_t count, const std::array<Word, kRootCount>& roots) {
t[slot(0)] = 1;
t[slot(0) + 8] = quotient(1);
for (int level = 0; level < kRootCount; ++level) {
const size_t h = size_t(1) << level;
if (h >= count) break;
const Word root = roots[level];
if (h < 8) {
for (size_t j = 0; j < h; ++j) {
const Word v = multiply_mod(t[slot(j)], root);
t[slot(h + j)] = v;
t[slot(h + j) + 8] = quotient(v);
}
continue;
}
const Factor by(root), to_r(kR);
for (size_t j = 0; j < h; j += 8) {
const Vec v = reduce(multiply(_mm256_load_si256(reinterpret_cast<const Vec*>(t + slot(j))), by), kP);
const Vec vr = reduce(multiply(v, to_r), kP);
_mm256_store_si256(reinterpret_cast<Vec*>(t + slot(h + j)), v);
_mm256_store_si256(reinterpret_cast<Vec*>(t + slot(h + j) + 8), _mm256_mullo_epi32(vr, broadcast(kernels::kNI)));
}
}
}
}
}
namespace blocked_ntt::kernels {
using namespace detail;
template<bool Inverse, bool Identity>
inline void radix4(Vec* a, size_t h, const Word* x, const Word* y) {
const Factor wx(Identity ? 1 : x[0], Identity ? 4 : x[8]);
const Factor wy(Identity ? 1 : y[0], Identity ? 4 : y[8]), wz(y[1], y[9]);
#pragma GCC unroll 1
for (size_t k = 0; k < h; ++k) {
const Vec f0 = a[k], f1 = a[k + h], f2 = a[k + 2 * h], f3 = a[k + 3 * h];
if constexpr (Inverse) {
const Vec ab = reduce(add(f0, f1), 2 * kP), cd = reduce(add(f2, f3), 2 * kP);
const Vec amb = Identity ? reduce(diff(f0, f1), 2 * kP) : multiply(diff(f0, f1), wy);
const Vec cmd = multiply(diff(f2, f3), wz);
a[k] = reduce(add(ab, cd), 2 * kP), a[k + h] = reduce(add(amb, cmd), 2 * kP);
if constexpr (Identity) {
a[k + 2 * h] = reduce(diff(ab, cd), 2 * kP);
a[k + 3 * h] = reduce(diff(amb, cmd), 2 * kP);
} else {
a[k + 2 * h] = multiply(diff(ab, cd), wx);
a[k + 3 * h] = multiply(diff(amb, cmd), wx);
}
} else {
const Vec lo = reduce(f0, 2 * kP), hi = reduce(f1, 2 * kP);
const Vec xc = Identity ? reduce(f2, 2 * kP) : multiply(f2, wx);
const Vec xd = Identity ? reduce(f3, 2 * kP) : multiply(f3, wx);
const Vec ac = reduce(add(lo, xc), 2 * kP), amc = reduce(diff(lo, xc), 2 * kP);
const Vec bd = Identity ? reduce(add(hi, xd), 2 * kP) : multiply(add(hi, xd), wy);
const Vec bmd = multiply(diff(hi, xd), wz);
a[k] = add(ac, bd), a[k + h] = diff(ac, bd);
a[k + 2 * h] = add(amc, bmd), a[k + 3 * h] = diff(amc, bmd);
}
}
}
inline void forward(Vec* a, size_t h, const Word* x, const Word* y) {
radix4<false, false>(a, h, x, y);
}
inline void inverse(Vec* a, size_t h, const Word* x, const Word* y) {
radix4<true, false>(a, h, x, y);
}
inline void forward_identity(Vec* a, size_t h, const Word* y) {
radix4<false, true>(a, h, y, y);
}
inline void inverse_identity(Vec* a, size_t h, const Word* y) {
radix4<true, true>(a, h, y, y);
}
}
namespace blocked_poly::kernels {
using Vec = __m256i;
inline constexpr Word kP = 998244353, k2P = 2 * kP;
[[gnu::noinline]] inline void forward_h4(Vec* a, size_t count, const Word* table, size_t k) {
for (size_t j = 0; j < count; ++j)
blocked_ntt::kernels::forward(a + 16 * j, 4, table + blocked_ntt::detail::slot(k + j), table + blocked_ntt::detail::slot(2 * (k + j)));
}
[[gnu::noinline]] inline void inverse_h4(Vec* a, size_t count, const Word* table, size_t k) {
for (size_t j = 0; j < count; ++j)
blocked_ntt::kernels::inverse(a + 16 * j, 4, table + blocked_ntt::detail::slot(k + j), table + blocked_ntt::detail::slot(2 * (k + j)));
}
}
namespace blocked_poly {
inline constexpr Word kModulus = blocked_ntt::kModulus;
enum class Half { kBoth, kLower, kUpper };
class Arena {
public:
explicit Arena(size_t words)
: storage_(allocate_fps_words(words)), next_(storage_.get()), end_(next_ + words) {}
Arena(const Arena &) = delete;
Arena &operator=(const Arena &) = delete;
static constexpr size_t footprint(size_t n) { return (n + 7) / 8 * 8 + 16; }
Words take(size_t n) {
if (size_t(end_ - next_) < footprint(n)) throw std::bad_alloc();
auto *start = next_;
next_ += footprint(n);
return {start, n};
}
bool zero_initialized() const { return storage_.get_deleter().region != nullptr; }
private:
FpsWords storage_;
Word *next_, *end_;
};
namespace detail {
using blocked_ntt::detail::add, blocked_ntt::detail::broadcast, blocked_ntt::detail::diff, blocked_ntt::detail::Factor, blocked_ntt::detail::kP, blocked_ntt::detail::kR, blocked_ntt::detail::reduce, blocked_ntt::detail::slot, blocked_ntt::detail::Vec;
inline constexpr auto kInverseScales = [] {
std::array<std::array<Word, blocked_ntt::kMaxLog + 1>, 2> s{};
for (int lg = 3; lg <= blocked_ntt::kMaxLog; ++lg) {
s[0][lg] = blocked_ntt::detail::power(Word(1) << (lg - 3), kP - 2);
s[1][lg] = blocked_ntt::detail::multiply_mod(s[0][lg], kR);
}
return s;
}();
inline Vec load(const Word* p) { return _mm256_load_si256(reinterpret_cast<const Vec*>(p)); }
inline void store(Word* p, Vec x) { _mm256_store_si256(reinterpret_cast<Vec*>(p), x); }
inline Vec low(Vec x) { return reduce(x, 2 * kP); }
inline Vec canonical(Vec x) { return reduce(reduce(x, 2 * kP), kP); }
inline Vec low_difference(Vec x, Vec y) {
const Vec t = _mm256_sub_epi32(x, y);
return _mm256_min_epu32(t, _mm256_add_epi32(t, broadcast(2 * kP)));
}
inline Vec difference(Vec x, Vec y) { return reduce(_mm256_sub_epi32(add(x, broadcast(kP)), y), kP); }
inline Vec times(Vec x, const Factor& w) { return blocked_ntt::detail::multiply(x, w); }
inline Factor entry(const Word* table, size_t k) {
const Word* e = table + slot(k);
return Factor(e[0], e[8]);
}
inline Factor leaf_weight(const Word* roots, size_t p) {
const Word* e = roots + slot(p >> 1);
return p & 1 ? Factor(kP - e[0], ~e[8]) : Factor(e[0], e[8]);
}
struct Group {
Factor x, y, z;
[[gnu::always_inline]] Group(const Word* table, size_t g)
: x(entry(table, g)), y(entry(table, 2 * g)), z(entry(table, 2 * g + 1)) {}
};
[[gnu::always_inline]] inline void forward_h1(Vec (&f)[4], const Group& w) {
const Vec a = low(f[0]), b = low(f[1]), c = times(f[2], w.x), d = times(f[3], w.x);
const Vec ac = low(add(a, c)), amc = low_difference(a, c);
const Vec bd = times(add(b, d), w.y), bmd = times(diff(b, d), w.z);
f[0] = add(ac, bd), f[1] = diff(ac, bd), f[2] = add(amc, bmd), f[3] = diff(amc, bmd);
}
[[gnu::always_inline]] inline void inverse_h1(Vec (&f)[4], const Group& w) {
const Vec ab = low(add(f[0], f[1])), cd = low(add(f[2], f[3]));
const Vec amb = times(diff(f[0], f[1]), w.y), cmd = times(diff(f[2], f[3]), w.z);
f[0] = low(add(ab, cd)), f[1] = low(add(amb, cmd));
f[2] = times(diff(ab, cd), w.x), f[3] = times(diff(amb, cmd), w.x);
}
struct alignas(64) Window {
Word word[17];
};
inline void fill_window(Window& window, Vec a, const Factor& w) {
window.word[16] = 0;
store(window.word + 8, a);
store(window.word, reduce(times(a, w), kP));
}
[[gnu::always_inline]] inline void fill_windows(Window (&window)[4], const Vec (&a)[4], const Word* roots,
size_t g) {
const Factor y = entry(roots, 2 * g), z = entry(roots, 2 * g + 1);
#pragma GCC unroll 4
for (int t = 0; t < 4; ++t) {
window[t].word[16] = 0;
store(window[t].word + 8, a[t]);
const Vec wa = times(a[t], t < 2 ? y : z);
store(window[t].word, reduce(t % 2 ? _mm256_sub_epi32(broadcast(2 * kP), wa) : wa, kP));
}
}
[[gnu::always_inline]] inline Vec leaf_product(const Window& window, const Word* b) {
Vec even = _mm256_setzero_si256(), odd = even;
#pragma GCC unroll 8
for (int i = 0; i < 8; ++i) {
const Vec bi = broadcast(b[i]);
const Vec lo = _mm256_loadu_si256(reinterpret_cast<const Vec*>(window.word + 8 - i));
const Vec hi = _mm256_loadu_si256(reinterpret_cast<const Vec*>(window.word + 9 - i));
even = _mm256_add_epi64(even, _mm256_mul_epu32(lo, bi));
odd = _mm256_add_epi64(odd, _mm256_mul_epu32(hi, bi));
}
return reduce(mint_detail::Ops::reduce8(even, odd), 2 * kP);
}
struct ForwardBottom {
static constexpr bool kForward = true, kInverse = false;
const Word* roots;
void operator()(Word* a, size_t count, size_t first) const {
for (size_t j = 0; j < count; ++j, a += 32) {
const Group w(roots, first + j);
Vec f[4] = {load(a), load(a + 8), load(a + 16), load(a + 24)};
forward_h1(f, w);
#pragma GCC unroll 4
for (int t = 0; t < 4; ++t) store(a + 8 * t, canonical(f[t]));
}
}
};
struct InverseBottom {
static constexpr bool kForward = false, kInverse = true;
const Word* inverse_roots;
const Word* in;
void operator()(Word* a, size_t count, size_t first) const {
const Word* from = in + 32 * first;
for (size_t j = 0; j < count; ++j, a += 32, from += 32) {
const Group w(inverse_roots, first + j);
Vec f[4] = {load(from), load(from + 8), load(from + 16), load(from + 24)};
inverse_h1(f, w);
for (int t = 0; t < 4; ++t) store(a + 8 * t, f[t]);
}
}
};
struct ProductBottom {
static constexpr bool kForward = true, kInverse = true;
const Word* roots;
const Word* inverse_roots;
const Word* b;
size_t origin = 0;
[[gnu::always_inline]] void prepare(const Word* a, size_t g, Window (&window)[4]) const {
Vec f[4] = {load(a), load(a + 8), load(a + 16), load(a + 24)};
forward_h1(f, Group(roots, g));
#pragma GCC unroll 4
for (int t = 0; t < 4; ++t) f[t] = canonical(f[t]);
fill_windows(window, f, roots, g);
}
[[gnu::always_inline]] void finish(Word* a, size_t g, const Window (&window)[4]) const {
Vec f[4];
#pragma GCC unroll 4
for (int t = 0; t < 4; ++t) f[t] = leaf_product(window[t], b + 8 * (4 * g + t - origin));
inverse_h1(f, Group(inverse_roots, g));
for (int t = 0; t < 4; ++t) store(a + 8 * t, f[t]);
}
void operator()(Word* a, size_t count, size_t first) const {
Window window[2][4];
prepare(a, first, window[0]);
for (size_t j = 0; j < count; ++j, a += 32) {
if (j + 1 < count) prepare(a + 32, first + j + 1, window[(j + 1) & 1]);
finish(a, first + j, window[j & 1]);
}
}
};
struct InverseProductBottom {
static constexpr bool kForward = false, kInverse = true;
ProductBottom product;
const Word* a;
[[gnu::always_inline]] void prepare(size_t g, Window (&window)[4]) const {
const Word* leaves = a + 32 * g;
const Vec f[4] = {load(leaves), load(leaves + 8), load(leaves + 16), load(leaves + 24)};
fill_windows(window, f, product.roots, g);
}
void operator()(Word* out, size_t count, size_t first) const {
Window window[2][4];
prepare(first, window[0]);
for (size_t j = 0; j < count; ++j, out += 32) {
if (j + 1 < count) prepare(first + j + 1, window[(j + 1) & 1]);
product.finish(out, first + j, window[j & 1]);
}
}
};
template <size_t K>
struct InverseProductSumBottom {
static constexpr bool kForward = false, kInverse = true;
std::array<InverseProductBottom, K> terms;
void prepare(size_t g, Window (&window)[K][4]) const {
for (size_t k = 0; k < K; ++k) terms[k].prepare(g, window[k]);
}
void operator()(Word* out, size_t count, size_t first) const {
Window window[2][K][4];
prepare(first, window[0]);
for (size_t j = 0; j < count; ++j, out += 32) {
if (j + 1 < count) prepare(first + j + 1, window[(j + 1) & 1]);
const size_t g = first + j;
Vec f[4];
#pragma GCC unroll 4
for (size_t t = 0; t < 4; ++t) {
f[t] = leaf_product(window[j & 1][0][t], terms[0].product.b + 8 * (4 * g + t - terms[0].product.origin));
for (size_t k = 1; k < K; ++k)
f[t] = low(add(f[t], leaf_product(window[j & 1][k][t], terms[k].product.b + 8 * (4 * g + t - terms[k].product.origin))));
}
inverse_h1(f, Group(terms[0].product.inverse_roots, g));
#pragma GCC unroll 4
for (size_t t = 0; t < 4; ++t) store(out + 8 * t, f[t]);
}
}
};
struct ForwardProductBottom {
static constexpr bool kForward = true, kInverse = false;
ProductBottom product;
void operator()(Word* a, size_t count, size_t first) const {
const Factor undo_montgomery(kR);
Window window[2][4];
product.prepare(a, first, window[0]);
for (size_t j = 0; j < count; ++j, a += 32) {
if (j + 1 < count) product.prepare(a + 32, first + j + 1, window[(j + 1) & 1]);
const Word* b = product.b + 8 * (4 * (first + j) - product.origin);
#pragma GCC unroll 4
for (int t = 0; t < 4; ++t)
store(a + 8 * t, reduce(times(leaf_product(window[j & 1][t], b + 8 * t), undo_montgomery), kP));
}
}
};
template <class Bottom>
class Recursion {
public:
Recursion(const Word* roots, const Word* inverse_roots, const Bottom& bottom)
: r_(roots), ir_(inverse_roots), bottom_(bottom) {}
void visit(Word* a, size_t nv, size_t k) const {
if (nv <= kTile) return tile(a, nv, k);
const size_t h = nv / 4;
if constexpr (Bottom::kForward) forward(a, h, k);
for (size_t t = 0; t < 4; ++t) visit(a + 8 * t * h, h, 4 * k + t);
if constexpr (Bottom::kInverse) inverse(a, h, k);
}
private:
static constexpr size_t kTile = 256;
void tile(Word* a, size_t nv, size_t k) const {
if constexpr (Bottom::kForward) {
for (size_t h = nv / 4; h >= 16; h /= 4)
for (size_t j = 0, g = k * (nv / (4 * h)); j < nv; j += 4 * h, ++g) forward(a + 8 * j, h, g);
if (nv >= 16) kernels::forward_h4(vectors(a), nv / 16, r_, k * (nv / 16));
}
bottom_(a, nv / 4, k * (nv / 4));
if constexpr (Bottom::kInverse) {
if (nv >= 16) kernels::inverse_h4(vectors(a), nv / 16, ir_, k * (nv / 16));
for (size_t h = 16; h < nv; h *= 4)
for (size_t j = 0, g = k * (nv / (4 * h)); j < nv; j += 4 * h, ++g) inverse(a + 8 * j, h, g);
}
}
static Vec* vectors(Word* a) { return reinterpret_cast<Vec*>(a); }
void forward(Word* a, size_t h, size_t k) const {
if (k == 0) return blocked_ntt::kernels::forward_identity(vectors(a), h, r_);
blocked_ntt::kernels::forward(vectors(a), h, r_ + slot(k), r_ + slot(2 * k));
}
void inverse(Word* a, size_t h, size_t k) const {
if (k == 0) return blocked_ntt::kernels::inverse_identity(vectors(a), h, ir_);
blocked_ntt::kernels::inverse(vectors(a), h, ir_ + slot(k), ir_ + slot(2 * k));
}
const Word *r_, *ir_;
Bottom bottom_;
};
class Source {
public:
Source(const Word* in, size_t size, size_t shift)
: in_(in), shift_(shift), end_(shift + size), first_((shift + 7) / 8), first_touched_(shift / 8) {
count_ = end_ / 8 > first_ ? end_ / 8 - first_ : 0;
touched_ = (end_ + 7) / 8 - first_touched_;
}
Vec operator()(size_t v) const {
if (v - first_ < count_) return _mm256_loadu_si256(reinterpret_cast<const Vec*>(in_ + (8 * v - shift_)));
if (v - first_touched_ >= touched_) return _mm256_setzero_si256();
return edge(in_, shift_, end_, v);
}
bool within(size_t lo, size_t hi) const {
return touched_ == 0 || (first_touched_ >= lo && first_touched_ + touched_ <= hi);
}
private:
[[gnu::noinline]] static Vec edge(const Word* in, size_t shift, size_t end, size_t v) {
alignas(32) Word x[8] = {};
for (size_t i = 8 * v; i < 8 * v + 8; ++i)
if (i >= shift && i < end) x[i - 8 * v] = in[i - shift];
return load(x);
}
const Word* in_;
size_t shift_, end_;
size_t first_, count_;
size_t first_touched_, touched_;
};
template <class Input>
[[gnu::always_inline]] inline void forward_top4_body(Input in, Vec* out, size_t h, const Word* roots) {
const Factor z(roots[1], roots[9]);
const Vec p = broadcast(kP);
if (in.within(0, 2 * h)) {
for (size_t j = 0; j < h; ++j) {
const Vec a = in(j), b = in(j + h), zb = times(b, z);
out[j] = add(a, b), out[j + h] = diff(a, b), out[j + 2 * h] = add(a, zb), out[j + 3 * h] = diff(a, zb);
}
return;
}
if (in.within(2 * h, 4 * h)) {
for (size_t j = 0; j < h; ++j) {
const Vec c = in(j + 2 * h), d = in(j + 3 * h);
const Vec mc = _mm256_sub_epi32(p, c), zmd = times(_mm256_sub_epi32(p, d), z);
out[j] = add(c, d), out[j + h] = diff(c, d), out[j + 2 * h] = add(mc, zmd), out[j + 3 * h] = diff(mc, zmd);
}
return;
}
for (size_t j = 0; j < h; ++j) {
const Vec a = in(j), b = in(j + h), c = in(j + 2 * h), d = in(j + 3 * h);
const Vec ac = add(a, c), amc = _mm256_sub_epi32(add(a, p), c);
const Vec bd = add(b, d), zbmd = times(_mm256_sub_epi32(add(b, p), d), z);
out[j] = add(ac, bd), out[j + h] = diff(ac, bd);
out[j + 2 * h] = add(amc, zbmd), out[j + 3 * h] = diff(amc, zbmd);
}
}
template <class Input>
[[gnu::always_inline]] inline void forward_top2_body(Input in, Vec* out, size_t h) {
const Vec p = broadcast(kP);
if (in.within(0, h)) {
for (size_t j = 0; j < h; ++j) out[j] = out[j + h] = in(j);
return;
}
if (in.within(h, 2 * h)) {
for (size_t j = 0; j < h; ++j) {
const Vec v = in(j + h);
out[j] = v, out[j + h] = _mm256_sub_epi32(p, v);
}
return;
}
for (size_t j = 0; j < h; ++j) {
const Vec u = in(j), v = in(j + h);
out[j] = add(u, v), out[j + h] = _mm256_sub_epi32(add(u, p), v);
}
}
[[gnu::noinline]] inline void forward_top4(Source in, Vec* out, size_t h, const Word* roots) {
forward_top4_body(in, out, h, roots);
}
[[gnu::noinline]] inline void forward_top2(Source in, Vec* out, size_t h) {
forward_top2_body(in, out, h);
}
template <class Bottom>
void radix2_halves(const Recursion<Bottom>& recursion, const Source& in, Word* a, size_t nv) {
const size_t h = nv / 2;
if constexpr (Bottom::kForward) forward_top2(in, reinterpret_cast<Vec*>(a), h);
recursion.visit(a, h, 0);
recursion.visit(a + 8 * h, h, 1);
}
inline void inverse_top4(Vec* f, size_t h, Half output, const Word* inverse_roots, const Factor& s) {
const Factor z(inverse_roots[1], inverse_roots[9]);
const auto scale = [&s](Vec x) __attribute__((target("avx2"))) { return reduce(times(x, s), kP); };
for (size_t j = 0; j < h; ++j) {
const Vec p0 = f[j], p1 = f[j + h], p2 = f[j + 2 * h], p3 = f[j + 3 * h];
const Vec ab = low(add(p0, p1)), cd = low(add(p2, p3));
const Vec amb = low(diff(p0, p1)), cmd = times(diff(p2, p3), z);
if (output != Half::kUpper) f[j] = scale(add(ab, cd)), f[j + h] = scale(add(amb, cmd));
if (output != Half::kLower) f[j + 2 * h] = scale(diff(ab, cd)), f[j + 3 * h] = scale(diff(amb, cmd));
}
}
inline void inverse_top2(Vec* f, size_t h, Half output, Word scale) {
const Factor s(scale);
for (size_t j = 0; j < h; ++j) {
const Vec lo = f[j], hi = f[j + h];
if (output != Half::kUpper) f[j] = reduce(times(add(lo, hi), s), kP);
if (output != Half::kLower) f[j + h] = reduce(times(diff(lo, hi), s), kP);
}
}
}
class Transform {
public:
static constexpr int kMinLog = 6, kMaxLog = blocked_ntt::kMaxLog;
static constexpr size_t words(int lg_max) { return Arena::footprint(2 * blocked_ntt::detail::table_words(lg_max)); }
Transform(Arena& arena, int lg_max) : lg_max_(lg_max) {
if (lg_max < kMinLog || lg_max > kMaxLog) std::abort();
const size_t table = blocked_ntt::detail::table_words(lg_max), entries = (size_t(1) << lg_max) / 16;
roots_ = arena.take(2 * table).data();
std::fill_n(roots_, 2 * table, 0u);
inverse_roots_ = roots_ + table;
blocked_ntt::detail::build_table(roots_, entries, blocked_ntt::detail::kRoots[0]);
blocked_ntt::detail::build_table(inverse_roots_, entries, blocked_ntt::detail::kRoots[1]);
}
int lg_max() const { return lg_max_; }
const Word* roots() const { return roots_; }
const Word* inverse_roots() const { return inverse_roots_; }
void forward(ConstWords in, size_t shift, Words out) const {
run(out, source(in, shift, out.size()), detail::ForwardBottom{roots_}, 0, Half::kBoth);
}
void forward(Words a) const { forward(a, 0, a); }
void forward_upper(ConstWords in, size_t shift, Words out) const {
using namespace detail;
check_length(2 * out.size());
const Source src = source(in, shift, out.size());
const detail::Recursion recursion(roots_, inverse_roots_, ForwardBottom{roots_});
const size_t nv = out.size() / 8;
auto* v = reinterpret_cast<Vec*>(out.data());
if (std::countr_zero(nv) % 2 == 0) {
for (size_t j = 0; j < nv; ++j) v[j] = src(j);
return recursion.visit(out.data(), nv, 1);
}
const size_t h = nv / 2;
const Factor z(roots_[1], roots_[9]);
for (size_t j = 0; j < h; ++j) {
const Vec a = src(j), zb = times(src(j + h), z);
v[j] = add(a, zb), v[j + h] = diff(a, zb);
}
recursion.visit(out.data(), h, 2);
recursion.visit(out.data() + 8 * h, h, 3);
}
void inverse(ConstWords in, Words out, Half output = Half::kBoth) const {
using namespace blocked_ntt::detail;
const Word scale = inverse_scale(out.size(), false);
run(out, detail::Source(nullptr, 0, 0), detail::InverseBottom{inverse_roots_, in.data()}, scale, output);
}
void inverse(Words a, Half output = Half::kBoth) const { inverse(a, a, output); }
void cyclic_product(ConstWords in, size_t shift, Words out,
ConstWords b, Half output = Half::kBoth, Word c = 1) const {
using namespace blocked_ntt::detail;
const Word scale = multiply_mod(inverse_scale(out.size(), true), c);
run(out, source(in, shift, out.size()), detail::ProductBottom{roots_, inverse_roots_, b.data()}, scale, output);
}
void cyclic_product(Words a, ConstWords b) const { cyclic_product(a, 0, a, b); }
void inverse_product(ConstWords a, ConstWords b, Words out,
Half output = Half::kBoth, Word c = 1) const {
using namespace blocked_ntt::detail;
const Word scale = multiply_mod(inverse_scale(out.size(), true), c);
const detail::ProductBottom product{roots_, inverse_roots_, b.data()};
run(out, detail::Source(nullptr, 0, 0), detail::InverseProductBottom{product, a.data()}, scale, output);
}
struct Pair {
ConstWords a, b;
};
void inverse_product_sum(std::span<const Pair> pairs, Words out, Half output = Half::kBoth) const {
switch (pairs.size()) {
case 1: return inverse_product(pairs[0].a, pairs[0].b, out, output);
case 2: return inverse_products<2>(pairs, out, output);
case 3: return inverse_products<3>(pairs, out, output);
default: std::abort();
}
}
template <class Bottom>
void inverse_with(const Bottom& bottom, Words out, Half output = Half::kBoth) const {
using namespace blocked_ntt::detail;
const Word scale = inverse_scale(out.size(), true);
run(out, detail::Source(nullptr, 0, 0), bottom, scale, output);
}
void forward_product(ConstWords in, size_t shift, Words out,
ConstWords b) const {
const detail::ProductBottom product{roots_, inverse_roots_, b.data()};
run(out, source(in, shift, out.size()), detail::ForwardProductBottom{product}, 0, Half::kBoth);
}
void multiply(Words a, ConstWords b) const {
products(a.data(), a.data(), b.data(), a.size(), [&](detail::Vec, detail::Vec p) __attribute__((target("avx2"))) { return p; });
}
void multiply_add(Words c, ConstWords a,
ConstWords b) const {
products(c.data(), a.data(), b.data(), a.size(), [&](detail::Vec c, detail::Vec p) __attribute__((target("avx2"))) { return detail::add(c, p); });
}
protected:
Word inverse_scale(size_t n, bool montgomery) const {
check_length(n);
return detail::kInverseScales[montgomery][std::countr_zero(n)];
}
void check_length(size_t n) const {
const int lg = std::countr_zero(n);
if (!std::has_single_bit(n) || lg < kMinLog || lg > lg_max_) std::abort();
}
template <size_t K>
void inverse_products(std::span<const Pair> pairs, Words out, Half output) const {
using namespace blocked_ntt::detail;
const Word scale = inverse_scale(out.size(), true);
detail::InverseProductSumBottom<K> bottom{};
for (size_t k = 0; k < K; ++k) bottom.terms[k] = {{roots_, inverse_roots_, pairs[k].b.data()}, pairs[k].a.data()};
run(out, detail::Source(nullptr, 0, 0), bottom, scale, output);
}
static detail::Source source(ConstWords in, size_t shift, size_t n) {
if (shift > n || in.size() > n - shift) std::abort();
return detail::Source(in.data(), in.size(), shift);
}
template <class Bottom>
void run(Words a, const detail::Source& in, const Bottom& bottom, Word scale,
Half output) const {
using namespace detail;
check_length(a.size());
const detail::Recursion recursion(roots_, inverse_roots_, bottom);
const size_t nv = a.size() / 8;
auto* v = reinterpret_cast<Vec*>(a.data());
if (std::countr_zero(nv) % 2 == 0) {
const size_t h = nv / 4;
if constexpr (Bottom::kForward) forward_top4(in, v, h, roots_);
for (size_t t = 0; t < 4; ++t) recursion.visit(a.data() + 8 * t * h, h, t);
if constexpr (Bottom::kInverse) inverse_top4(v, h, output, inverse_roots_, Factor(scale));
} else {
const size_t h = nv / 2;
radix2_halves(recursion, in, a.data(), nv);
if constexpr (Bottom::kInverse) inverse_top2(v, h, output, scale);
}
}
template <class Combine>
void products(Word* out, const Word* a, const Word* b, size_t n, Combine combine) const {
using namespace detail;
check_length(n);
const Factor undo_montgomery(kR);
Window window[2];
fill_window(window[0], load(a), leaf_weight(roots_, 0));
for (size_t p = 0; p < n / 8; ++p) {
if (p + 1 < n / 8) fill_window(window[(p + 1) & 1], load(a + 8 * p + 8), leaf_weight(roots_, p + 1));
const Vec product = reduce(times(leaf_product(window[p & 1], b + 8 * p), undo_montgomery), kP);
store(out + 8 * p, canonical(combine(load(out + 8 * p), product)));
}
}
int lg_max_;
Word *roots_, *inverse_roots_;
};
namespace detail {
inline Vec load_unaligned(const Word* p) { return _mm256_loadu_si256(reinterpret_cast<const Vec*>(p)); }
inline void store_unaligned(Word* p, Vec x) { _mm256_storeu_si256(reinterpret_cast<Vec*>(p), x); }
inline Vec montgomery(Vec a, Vec b) { return mint_detail::Ops::mul8(a, b); }
class Indices {
public:
explicit Indices(size_t k) {
alignas(32) Word x[8];
for (Word i = 0; i < 8; ++i) x[i] = Word((std::uint64_t(k + i) << 32) % kP);
value_ = load(x);
}
Vec value() const { return value_; }
void next() { value_ = reduce(add(value_, broadcast(kStep)), kP); }
private:
static constexpr Word kStep = Word((std::uint64_t(8) << 32) % kP);
Vec value_;
};
inline Word scalar_inverse(Word x) { return blocked_ntt::detail::power(x, kP - 2); }
template <class A>
void divide_by_index(size_t first, Words out, const A& a, Word factor = 1) {
constexpr size_t kLanes = 32;
const size_t n = out.size();
if (n == 0) return;
const size_t blocks = (n + kLanes - 1) / kLanes, last = kLanes * (blocks - 1);
const Vec one = broadcast(1), step = broadcast(kLanes);
Vec x[4];
for (int v = 0; v < 4; ++v)
x[v] = _mm256_add_epi32(broadcast(Word(first + 8 * v)), _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7));
const auto valid = [n](size_t at) __attribute__((target("avx2"))) {
const Vec lanes = _mm256_add_epi32(broadcast(Word(at)), _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7));
return _mm256_cmpgt_epi32(broadcast(Word(n)), lanes);
};
const auto masked = [&](Vec value, size_t at) __attribute__((target("avx2"))) { return at + 8 <= n ? value : _mm256_blendv_epi8(one, value, valid(at)); };
Vec prefix[4];
for (int v = 0; v < 4; ++v) prefix[v] = masked(x[v], 8 * v);
for (size_t b = kLanes; b < n; b += kLanes) {
for (int v = 0; v < 4; ++v) {
store_unaligned(out.data() + b - kLanes + 8 * v, prefix[v]);
x[v] = _mm256_add_epi32(x[v], step);
prefix[v] = montgomery(prefix[v], masked(x[v], b + 8 * v));
}
}
const Vec left = montgomery(prefix[0], prefix[1]), right = montgomery(prefix[2], prefix[3]);
alignas(32) Word total[8], inverse[8];
store(total, canonical(montgomery(left, right)));
Word running = 1;
for (size_t l = 0; l < 8; ++l) {
inverse[l] = running;
running = blocked_ntt::detail::multiply_mod(running, total[l]);
}
running = blocked_ntt::detail::multiply_mod(scalar_inverse(running), blocked_ntt::detail::multiply_mod(kR, factor));
for (size_t l = 8; l-- > 0;) {
inverse[l] = blocked_ntt::detail::multiply_mod(inverse[l], running);
running = blocked_ntt::detail::multiply_mod(running, total[l]);
}
const Vec inv_left = montgomery(load(inverse), right), inv_right = montgomery(load(inverse), left);
Vec inv[4] = {montgomery(inv_left, prefix[1]), montgomery(inv_left, prefix[0]),
montgomery(inv_right, prefix[3]), montgomery(inv_right, prefix[2])};
const auto put = [&](size_t at, Vec reciprocal) __attribute__((target("avx2"))) {
const Vec q = canonical(montgomery(reciprocal, a(at)));
if (at + 8 <= n) return store_unaligned(out.data() + at, q);
_mm256_maskstore_epi32(reinterpret_cast<int*>(out.data() + at), valid(at), q);
};
for (size_t b = last; b > 0; b -= kLanes) {
for (int v = 0; v < 4; ++v) {
const size_t at = b + 8 * v;
if (at < n) put(at, montgomery(load_unaligned(out.data() + at - kLanes), inv[v]));
inv[v] = montgomery(inv[v], masked(x[v], at));
x[v] = _mm256_sub_epi32(x[v], step);
}
}
for (int v = 0; v < 4; ++v)
if (8 * size_t(v) < n) put(8 * v, inv[v]);
}
}
inline Word mint_scalar(Word x) { return x; }
inline unsigned mint_scalar(mint x) { return std::min(x.v, x.v - mint::mod()); }
inline void copy_coefficients(const mint *in, Words out) {
for (size_t i = 0; i < out.size(); ++i) out[i] = mint_scalar(in[i]);
}
template <class T> inline detail::Vec coefficients(std::span<const T> f, size_t at) {
detail::Vec value;
std::memcpy(&value, f.data() + at, sizeof(value));
if constexpr (std::is_same_v<T, mint>) return detail::reduce(value, detail::kP);
else return value;
}
template <class T> inline void derivative_impl(std::span<const T> f, Words d) {
using namespace detail;
const size_t n = d.size(), full = std::min(n, f.size() > 0 ? f.size() - 1 : 0) / 8 * 8;
Indices index(1);
for (size_t i = 0; i < full; i += 8, index.next())
store_unaligned(d.data() + i, canonical(montgomery(coefficients(f, i + 1), index.value())));
for (size_t i = full; i < n; ++i)
d[i] = i + 1 < f.size() ? blocked_ntt::detail::multiply_mod(Word(i + 1), mint_scalar(f[i + 1])) : 0;
}
inline void derivative(ConstWords f, Words d) { derivative_impl(f, d); }
inline void divide_by_index(ConstWords a, size_t first, Words q) {
if (a.size() != q.size() || first == 0 || first + a.size() > kModulus) std::abort();
const size_t n = a.size();
detail::divide_by_index(first, q, [a, n](size_t at) __attribute__((target("avx2"))) {
if (at + 8 <= n) return detail::load_unaligned(a.data() + at);
alignas(32) Word rest[8] = {};
std::copy(a.begin() + at, a.end(), rest);
return detail::load(rest);
});
}
namespace detail {
inline constexpr size_t kInverseBase = 32;
inline Word direct_product(const Word* a, const Word* b, size_t n) {
std::uint64_t sums[4] = {};
size_t j = 0;
for (; j + 4 <= n; j += 4)
for (size_t k = 0; k < 4; ++k) sums[k] += std::uint64_t(a[j + k]) * b[n - 1 - j - k];
std::uint64_t sum = 0;
for (auto x : sums) sum += x % kP;
for (; j < n; ++j) sum += std::uint64_t(a[j]) * b[n - 1 - j];
return Word(sum % kP);
}
inline void inverse_direct(ConstWords f, Words g, Word inv0 = 0) {
using blocked_ntt::detail::multiply_mod;
if (!inv0) inv0 = blocked_ntt::detail::power(f[0], kP - 2);
g[0] = inv0;
for (size_t i = 1; i < g.size(); ++i) {
const size_t count = std::min(i, f.size() - 1);
const Word sum = direct_product(f.data() + 1, g.data() + i - count, count);
g[i] = multiply_mod(sum ? kP - sum : 0, inv0);
}
}
}
inline int inverse_log(size_t n) {
return std::max(Transform::kMinLog, int(std::bit_width(std::max<size_t>(n, 2) - 1)));
}
inline size_t inverse_scratch(size_t n) { return 2 * Arena::footprint(size_t(1) << inverse_log(n)); }
inline void inverse_step(const Transform& t, ConstWords f, ConstWords g,
Words to, Words gt, Words work, Word scale = detail::kP - 1) {
const size_t k = g.size();
t.forward(g, 0, gt);
t.cyclic_product(f.first(std::min(2 * k, f.size())), 0, work, gt, Half::kUpper);
t.cyclic_product(work.subspan(k), k, work, gt, Half::kUpper, scale);
if (to.data() != work.data() + k) std::copy_n(work.begin() + std::ptrdiff_t(k), to.size(), to.begin());
}
inline void inverse(const Transform& t, ConstWords f, Words g,
Words scratch) {
const size_t n = g.size();
size_t k = std::min(n, detail::kInverseBase);
detail::inverse_direct(f, g.first(k));
for (; k < n; k *= 2) {
const size_t len = 2 * k;
inverse_step(t, f, g.first(k), g.subspan(k, std::min(k, n - k)), scratch.first(len),
scratch.subspan(Arena::footprint(len), len));
}
}
namespace detail {
inline constexpr size_t kLogBase = 64;
inline constexpr size_t kLogBlocks = 4;
inline void log_direct(ConstWords f, Words g) {
using blocked_ntt::detail::multiply_mod;
Word a[kLogBase] = {}, inv[kLogBase] = {0, 1}, kg[kLogBase] = {};
std::copy_n(f.begin(), std::min(f.size(), g.size()), a);
for (size_t i = 2; i < g.size(); ++i) inv[i] = multiply_mod(kP - kP / Word(i), inv[kP % i]);
g[0] = 0;
for (size_t i = 1; i < g.size(); ++i) {
const Word sum = direct_product(kg + 1, a + 1, i - 1);
kg[i] = (multiply_mod(Word(i), a[i]) + kP - sum) % kP;
g[i] = multiply_mod(kg[i], inv[i]);
}
}
inline size_t log_block(size_t n) {
return std::max<size_t>(32, std::bit_ceil((n - 1 + kLogBlocks - 1) / kLogBlocks));
}
inline size_t log_blocks(size_t n) { return (n - 1 + log_block(n) - 1) / log_block(n); }
template <class T> inline void subtract_derivative(std::span<const T> f, size_t first, size_t count,
Word* out, size_t size) {
const size_t inside = f.size() > first + 1 ? std::min(count, f.size() - first - 1) : 0, full = inside / 8 * 8;
Indices index(first + 1);
for (size_t c = 0; c < full; c += 8, index.next()) {
const Vec d = canonical(montgomery(coefficients(f, first + 1 + c), index.value()));
store_unaligned(out + c, reduce(_mm256_sub_epi32(add(d, broadcast(kP)), load_unaligned(out + size + c)), kP));
}
for (size_t c = full; c < count; ++c) {
const Word d = c < inside ? blocked_ntt::detail::multiply_mod(Word(first + 1 + c), mint_scalar(f[first + 1 + c])) : 0;
out[c] = (d + kP - out[size + c]) % kP;
}
}
inline void subtract_derivative_impl(ConstWords f, size_t first, size_t count, Word* out, size_t size) {
subtract_derivative(f, first, count, out, size);
}
inline void subtract_from_derivative(ConstWords f, size_t first, size_t count, Word* out, size_t size) {
subtract_derivative_impl(f, first, count, out, size);
}
struct ToeplitzBottom {
static constexpr bool kForward = false, kInverse = true;
const Word *roots, *inverse_roots, *w1, *w2, *q0, *q1;
Word* w3;
[[gnu::always_inline]] void prepare(size_t g, Window (&window)[3][4]) const {
Vec sum[4], d1[4], d3[4];
#pragma GCC unroll 4
for (size_t t = 0; t < 4; ++t) {
const size_t leaf = 8 * (4 * g + t);
const Vec v2 = load(w2 + leaf);
sum[t] = reduce(add(load(q0 + leaf), load(q1 + leaf)), kP);
d1[t] = difference(load(w1 + leaf), v2);
d3[t] = difference(load(w3 + leaf), v2);
}
fill_windows(window[0], sum, roots, g);
fill_windows(window[1], d1, roots, g);
fill_windows(window[2], d3, roots, g);
}
void operator()(Word* out, size_t count, size_t first) const {
Window window[2][3][4];
prepare(first, window[0]);
for (size_t j = 0; j < count; ++j, out += 32) {
if (j + 1 < count) prepare(first + j + 1, window[(j + 1) & 1]);
const size_t g = first + j;
const auto& w = window[j & 1];
Vec f[4];
#pragma GCC unroll 4
for (size_t t = 0; t < 4; ++t) {
const size_t leaf = 8 * (4 * g + t);
const Vec m1 = leaf_product(w[0][t], w2 + leaf);
f[t] = low(add(m1, leaf_product(w[1][t], q1 + leaf)));
store(w3 + leaf, low(add(m1, leaf_product(w[2][t], q0 + leaf))));
}
inverse_h1(f, Group(inverse_roots, g));
#pragma GCC unroll 4
for (size_t t = 0; t < 4; ++t) store(out + 8 * t, f[t]);
}
}
};
struct SideProductBottom {
static constexpr bool kForward = false, kInverse = true;
InverseProductBottom term;
const Word* side;
void operator()(Word* out, size_t count, size_t first) const {
Window window[2][4];
term.prepare(first, window[0]);
for (size_t j = 0; j < count; ++j, out += 32) {
if (j + 1 < count) term.prepare(first + j + 1, window[(j + 1) & 1]);
const size_t g = first + j;
Vec f[4];
#pragma GCC unroll 4
for (size_t t = 0; t < 4; ++t) {
const size_t leaf = 8 * (4 * g + t);
f[t] = low(add(leaf_product(window[j & 1][t], term.product.b + (leaf - 8 * term.product.origin)), load(side + leaf)));
}
inverse_h1(f, Group(term.product.inverse_roots, g));
#pragma GCC unroll 4
for (size_t t = 0; t < 4; ++t) store(out + 8 * t, f[t]);
}
}
};
inline size_t log_buffers(size_t n) { return std::max<size_t>(log_blocks(n) + 2, 3); }
inline Vec mint_load(std::span<const mint> f, size_t at) {
if (at + 8 <= f.size()) {
Vec value;
std::memcpy(&value, f.data() + at, sizeof(value));
return reduce(value, kP);
}
alignas(32) unsigned values[8] = {};
for (size_t i = 0; i < 8 && at + i < f.size(); ++i) values[i] = mint_scalar(f[at + i]);
return load(values);
}
class MintSource {
public:
explicit MintSource(std::span<const mint> f) : f_(f), full_(f.size() / 8), touched_((f.size() + 7) / 8) {}
Vec operator()(size_t v) const {
if (v < full_) {
Vec value;
std::memcpy(&value, f_.data() + 8 * v, sizeof(value));
return reduce(value, kP);
}
if (v >= touched_) return _mm256_setzero_si256();
return edge(f_, 8 * v);
}
bool within(size_t lo, size_t hi) const { return touched_ == 0 || (lo == 0 && touched_ <= hi); }
private:
[[gnu::noinline]] static Vec edge(std::span<const mint> f, size_t at) { return mint_load(f, at); }
std::span<const mint> f_;
size_t full_, touched_;
};
[[gnu::noinline]] inline void mint_forward_top4(MintSource in, Vec* out, size_t h, const Word* roots) {
forward_top4_body(in, out, h, roots);
}
[[gnu::noinline]] inline void mint_forward_top2(MintSource in, Vec* out, size_t h) {
forward_top2_body(in, out, h);
}
inline void mint_forward(const Transform& t, std::span<const mint> f, Words out) {
const MintSource in(f);
const detail::Recursion recursion(t.roots(), t.inverse_roots(), ForwardBottom{t.roots()});
const size_t nv = out.size() / 8;
auto* v = reinterpret_cast<Vec*>(out.data());
if (std::countr_zero(nv) % 2 == 0) {
const size_t h = nv / 4;
mint_forward_top4(in, v, h, t.roots());
for (size_t q = 0; q < 4; ++q) recursion.visit(out.data() + 8 * q * h, h, q);
} else {
const size_t h = nv / 2;
mint_forward_top2(in, v, h);
recursion.visit(out.data(), h, 0);
recursion.visit(out.data() + 8 * h, h, 1);
}
}
inline void mint_derivative(std::span<const mint> f, Words d) { derivative_impl(f, d); }
inline void mint_subtract_from_derivative(std::span<const mint> f, size_t first, size_t count, Word* out, size_t size) {
subtract_derivative(f, first, count, out, size);
}
struct NoLogTransform {
void operator()(size_t, ConstWords) const {}
};
template <class Input, class Sink, class Transformed>
[[gnu::always_inline]] inline void log_derivative(const Transform& t, Input f, size_t n,
Words scratch, ConstWords prefix, const Sink& sink, const Transformed& transformed = {}) {
const size_t k = log_block(n), len = 2 * k, blocks = log_blocks(n);
const auto buffer = [&scratch, len](size_t i) __attribute__((target("avx2"))) { return scratch.subspan(i * Arena::footprint(len), len); };
const Words ht = buffer(0), q0 = buffer(blocks);
const auto window = [&buffer](size_t t) __attribute__((target("avx2"))) { return buffer(t); };
inverse(t, prefix, ht.first(k), scratch.subspan(Arena::footprint(len), inverse_scratch(k)));
t.forward(ht.first(k), 0, ht);
for (size_t s = 1; s < blocks; ++s) {
const size_t from = std::min(f.size(), (s - 1) * k);
if constexpr (std::is_same_v<Input, std::span<const mint>>) mint_forward(t, f.subspan(from, std::min(f.size() - from, len)), window(s));
else t.forward(f.subspan(from, std::min(f.size() - from, len)), 0, window(s));
}
for (size_t j = 0; j < blocks; ++j) {
const size_t first = j * k, count = std::min(k, n - 1 - first);
const Words work = j == 0 ? q0 : buffer(blocks + 1);
if (j == 0) {
if constexpr (std::is_same_v<Input, std::span<const mint>>) mint_derivative(f.first(std::min(f.size(), count + 1)), work.first(count));
else derivative_impl(f.first(std::min(f.size(), count + 1)), work.first(count));
} else {
if (j == 1) {
t.inverse_product(window(1), q0, work, Half::kUpper);
} else if (blocks == 3) {
const Transform::Pair pairs[2] = {{window(2), q0}, {window(1), work}};
t.inverse_product_sum(pairs, work, Half::kUpper);
} else if (j == 2) {
t.inverse_with(ToeplitzBottom{t.roots(), t.inverse_roots(), window(1).data(), window(2).data(), q0.data(),
work.data(), window(3).data()},
work, Half::kUpper);
} else {
const InverseProductBottom product{{t.roots(), t.inverse_roots(), work.data()}, window(1).data()};
t.inverse_with(SideProductBottom{product, window(3).data()}, work, Half::kUpper);
}
if constexpr (std::is_same_v<Input, std::span<const mint>>) mint_subtract_from_derivative(f, first, count, work.data(), k);
else subtract_derivative_impl(f, first, count, work.data(), k);
}
t.cyclic_product(work.first(count), 0, work, ht, Half::kLower);
sink(first, ConstWords(work.first(count)));
if (j + 1 < blocks) {
t.forward(work.first(k), 0, work);
transformed(j, ConstWords(work));
}
}
}
template <class Sink, class Transformed = NoLogTransform>
[[gnu::always_inline]] inline void log_derivative(const Transform& t, ConstWords f, size_t n,
Words scratch, const Sink& sink, const Transformed& transformed = {}) {
log_derivative(t, f, n, scratch, f, sink, transformed);
}
template <class Sink, class Transformed = NoLogTransform>
[[gnu::always_inline]] inline void mint_log_derivative(const Transform& t, std::span<const mint> f, size_t n,
Words scratch, ConstWords prefix, const Sink& sink, const Transformed& transformed = {}) {
log_derivative(t, f, n, scratch, prefix, sink, transformed);
}
inline int log_derivative_log(size_t n) { return std::countr_zero(2 * log_block(n)); }
inline size_t log_derivative_scratch(size_t n) { return log_buffers(n) * Arena::footprint(2 * log_block(n)); }
}
inline int log_log(size_t n) {
return n <= detail::kLogBase ? Transform::kMinLog : std::countr_zero(2 * detail::log_block(n));
}
inline size_t log_scratch(size_t n) {
return n <= detail::kLogBase ? 0 : detail::log_buffers(n) * Arena::footprint(2 * detail::log_block(n));
}
inline void log(const Transform& t, ConstWords f, Words g,
Words scratch) {
using namespace detail;
const size_t n = g.size();
if (n <= kLogBase) return log_direct(f, g);
log_derivative(t, f, n, scratch, [g](size_t first, ConstWords q) __attribute__((target("avx2"))) {
detail::divide_by_index(first + 1, g.subspan(first + 1, q.size()), [q](size_t i) __attribute__((target("avx2"))) { return load(q.data() + i); });
});
g[0] = 0;
}
inline void native_log(const mint *in, size_t input, std::vector<mint> &out, size_t n, bool integrate) {
prepare_fps_output(out, n);
out.emplace_back(0);
const size_t c = std::min(input, n);
const int lg = detail::log_derivative_log(n);
const bool direct = n >= 262144;
const auto prefix_size = direct ? std::min(c, detail::log_block(n)) : c;
Arena arena(Transform::words(lg) + Arena::footprint(prefix_size) + Arena::footprint(log_scratch(n)));
const Transform t(arena, lg);
auto f = arena.take(prefix_size), scratch = arena.take(log_scratch(n));
copy_coefficients(in, f);
const auto sink = [&out, integrate, n](size_t first, ConstWords q) __attribute__((target("avx2"))) {
if (integrate) {
auto *work = const_cast<Word *>(q.data()) + detail::log_block(n);
detail::divide_by_index(first + 1, {work, q.size()}, [q](size_t i) __attribute__((target("avx2"))) { return detail::load(q.data() + i); }, blocked_ntt::detail::kR);
}
const auto *values = integrate ? q.data() + detail::log_block(n) : q.data();
if (integrate) append_fps_words<true>(out, values, q.size());
else append_fps_words<false>(out, values, q.size());
};
if (direct) detail::mint_log_derivative(t, {in, c}, n, scratch, f, sink);
else detail::log_derivative(t, f, n, scratch, sink);
}
}
}
#pragma GCC pop_options
#endif
#line 6 "components/fps_division_blocked.hpp"
#if defined(__linux__) && defined(__x86_64__) && defined(__GNUC__) && !defined(__clang__)
#pragma GCC push_options
_Pragma("GCC target(\"avx2\")")
#pragma GCC optimize("O3")
namespace fps_detail::blocked_poly {
inline size_t quotient_size(size_t n, size_t m) { return n >= m ? n - m + 1 : 0; }
inline size_t remainder_size(size_t n, size_t m) { return std::min(n, m - 1); }
namespace division_detail {
using namespace detail;
inline constexpr size_t kDivisionTerms = 8;
inline constexpr size_t kLongDivision = size_t(1) << 12;
inline constexpr size_t kDirectRemainder = 64;
inline constexpr std::array<double, 26> kTransformNs = {0, 0, 0, 0, 0, 0, 0.45, 0.48, 0.51,
0.55, 0.6, 0.65, 0.7, 0.76, 0.81, 0.88, 0.95, 1.02,
1.09, 1.17, 1.23, 1.3, 1.37, 1.44, 1.51, 1.58};
inline constexpr double kLeafProductNs = 0.92, kDirectTermNs = 0.04, kFoldNs = 0.25;
inline size_t transform_length(size_t n) { return std::max<size_t>(64, std::bit_ceil(n)); }
inline double transform_ns(size_t len) {
const auto lg = std::countr_zero(len);
return size_t(lg) < kTransformNs.size() ? kTransformNs[lg] * double(len) : std::numeric_limits<double>::infinity();
}
inline double cyclic_ns(size_t len) { return 2 * transform_ns(len) + kLeafProductNs * double(len); }
inline double inverse_ns(size_t n) {
double ns = 0;
for (size_t k = kInverseBase; k < n; k *= 2) ns += transform_ns(2 * k) + 2 * cyclic_ns(2 * k);
return ns;
}
struct QuotientPlan {
size_t s = 0, blocks = 0, pad = 0, windows = 0, tail = 0;
bool tails = false;
double ns = std::numeric_limits<double>::infinity();
QuotientPlan() = default;
QuotientPlan(size_t k, size_t d, size_t block)
: s(block), blocks((k + block - 1) / block), pad(blocks * block - k), tail(transform_length(2 * d)),
tails(d < block) {
const size_t len = 2 * s;
ns = inverse_ns(precision(k)) + transform_ns(len) + double(blocks) * cyclic_ns(len);
if (blocks == 1) return;
if (tails) {
ns += transform_ns(tail) + double(blocks - 1) * cyclic_ns(tail);
return;
}
const size_t reach = d / s + 1;
windows = std::min(reach, blocks - 1);
if (windows > kDivisionTerms) {
ns = std::numeric_limits<double>::infinity();
return;
}
double terms = 0;
for (size_t j = 1; j < blocks; ++j) terms += double(std::min(j, reach));
ns += double(windows + 2 * (blocks - 1)) * transform_ns(len) + terms * kLeafProductNs * double(len);
}
size_t precision(size_t k) const { return blocks == 1 ? k : s; }
};
enum class Remainder { kDirect, kWrap, kSplit, kTail };
struct RemainderPlan {
Remainder method;
size_t length = 0;
double ns = std::numeric_limits<double>::infinity();
RemainderPlan(Remainder how, size_t n, size_t k, size_t d, const QuotientPlan& quotient) : method(how) {
const size_t wrap = transform_length(d);
switch (how) {
case Remainder::kDirect:
if (k <= kDirectRemainder) ns = double(k) * double(d) * kDirectTermNs;
break;
case Remainder::kWrap:
length = wrap;
ns = cyclic_ns(length) + transform_ns(length) + double(n + k) * kFoldNs;
break;
case Remainder::kSplit:
length = wrap / 2;
if (length >= 64 && d > length && k + (d - length) <= length + 1)
ns = transform_ns(length) + 2 * cyclic_ns(length) + double(n + d) * kFoldNs;
break;
case Remainder::kTail:
if (!quotient.tails) break;
ns = (quotient.blocks == 1 ? transform_ns(quotient.tail) : 0) + cyclic_ns(quotient.tail) + double(d) * kFoldNs;
break;
}
}
size_t buffers() const { return method == Remainder::kWrap ? 2 : method == Remainder::kSplit ? 3 : 0; }
};
struct DivisionPlan {
size_t n, m, k, d;
bool long_division;
QuotientPlan quotient;
RemainderPlan remainder;
int lg = Transform::kMinLog;
DivisionPlan(size_t n_, size_t m_)
: n(n_), m(m_), k(quotient_size(n_, m_)), d(m_ - 1), long_division(k * d <= kLongDivision),
remainder(Remainder::kWrap, n, k, d, quotient) {
if (long_division) return;
double best = std::numeric_limits<double>::infinity();
for (size_t s = 32;; s *= 2) {
const QuotientPlan p(k, d, s);
for (Remainder how : {Remainder::kDirect, Remainder::kWrap, Remainder::kSplit, Remainder::kTail}) {
const RemainderPlan r(how, n, k, d, p);
if (p.ns + r.ns < best) best = p.ns + r.ns, quotient = p, remainder = r;
}
if (s >= k || 4 * s > (size_t(1) << Transform::kMaxLog)) break;
}
set_log();
}
DivisionPlan(size_t n_, size_t m_, size_t s, Remainder how)
: n(n_), m(m_), k(quotient_size(n_, m_)), d(m_ - 1), long_division(false), quotient(k, d, s),
remainder(how, n, k, d, quotient) {
set_log();
}
bool tails() const { return quotient.tails && (quotient.blocks > 1 || remainder.method == Remainder::kTail); }
void set_log() { lg = std::countr_zero(std::max({2 * quotient.s, tails() ? quotient.tail : 0, remainder.length})); }
size_t divisor() const {
const QuotientPlan& p = quotient;
return tails() ? m : std::min(m, std::max(p.precision(k), (p.windows + 1) * p.s));
}
size_t quotient_words() const {
const QuotientPlan& p = quotient;
const size_t len = 2 * p.s;
size_t later = Arena::footprint(p.blocks * p.s + p.s);
if (tails()) later += 2 * Arena::footprint(p.tail);
if (p.windows) later += (2 * p.windows + 1) * Arena::footprint(len);
return Arena::footprint(divisor()) + Arena::footprint(len) + std::max(inverse_scratch(p.precision(k)), later);
}
size_t scratch() const { return std::max(quotient_words(), remainder.buffers() * Arena::footprint(remainder.length)); }
size_t words() const {
return long_division ? Arena::footprint(n) : Transform::words(lg) + Arena::footprint(scratch());
}
};
class Bump {
public:
explicit Bump(Words memory) : memory_(memory) {}
Words take(size_t n) {
if (used_ > memory_.size() || Arena::footprint(n) > memory_.size() - used_) throw std::bad_alloc();
const Words s = memory_.subspan(used_, n);
used_ += Arena::footprint(n);
return s;
}
Words rest() const { return memory_.subspan(used_); }
private:
Words memory_;
size_t used_ = 0;
};
inline Vec montgomery_reduce(Vec even, Vec odd) {
return mint_detail::Ops::reduce8(even, odd);
}
inline Word input_word(Word x) { return x; }
inline Word input_word(mint x) { return std::min(x.v, x.v - mint::mod()); }
inline Vec load_any(const Word* p) { return _mm256_loadu_si256(reinterpret_cast<const Vec*>(p)); }
inline Vec load_any(const mint* p) {
static_assert(sizeof(mint) == sizeof(Word) && std::is_trivially_copyable_v<mint>);
Vec value;
std::memcpy(&value, p, sizeof(value));
return reduce(value, kP);
}
inline void store_any(Word* p, Vec x) { _mm256_storeu_si256(reinterpret_cast<Vec*>(p), x); }
inline Vec reversed(Vec x) { return _mm256_permutevar8x32_epi32(x, _mm256_setr_epi32(7, 6, 5, 4, 3, 2, 1, 0)); }
inline void append_words(std::vector<mint>& out, const Word* values, size_t n, Word factor = 1) {
const Factor scale(factor);
size_t i = 0;
for (; i + 8 <= n; i += 8) {
Vec word = load_any(values + i);
if (factor != 1) word = times(word, scale);
mint block[8];
static_assert(sizeof(block) == sizeof(word) && std::is_trivially_copyable_v<mint>);
std::memcpy(static_cast<void*>(block), &word, sizeof(block));
out.insert(out.end(), block, block + 8);
}
for (; i < n; ++i) out.emplace_back(mint::word(factor == 1 ? values[i] : blocked_ntt::detail::multiply_mod(values[i], factor)));
}
inline void append_division_words(std::vector<mint>& out, const Word* values, size_t n, Word factor = 1) {
static_assert(sizeof(mint) == sizeof(Word) && std::is_trivially_copyable_v<mint>);
const Factor scale(factor);
std::array<mint, 64> batch;
size_t i = 0;
for (; i + 64 <= n; i += 64) {
if (factor == 1) std::memcpy(static_cast<void*>(batch.data()), values + i, sizeof(batch));
else {
#pragma GCC unroll 8
for (size_t j = 0; j < 64; j += 8) {
const Vec word = times(load_any(values + i + j), scale);
std::memcpy(static_cast<void*>(batch.data() + j), &word, sizeof(word));
}
}
out.insert(out.end(), batch.begin(), batch.end());
}
if (i < n) append_words(out, values + i, n - i, factor);
}
template<class T>
inline void subtract(const T* a, const Word* b, Word* out, size_t count) {
size_t i = 0;
for (; i + 8 <= count; i += 8) store_any(out + i, difference(load_any(a + i), load_any(b + i)));
for (; i < count; ++i) out[i] = (input_word(a[i]) + kP - b[i]) % kP;
}
template<class T>
inline void reverse_difference(std::span<T> a, std::ptrdiff_t top, Word* out, size_t count,
const Word* sub = nullptr) {
const std::ptrdiff_t size = std::ptrdiff_t(a.size());
const auto value = [&](size_t i) {
const std::ptrdiff_t p = top - std::ptrdiff_t(i);
const Word x = p >= 0 && p < size ? input_word(a[size_t(p)]) : 0;
return sub ? (x + kP - sub[i]) % kP : x;
};
size_t i = top >= size ? std::min(count, size_t(top - size + 1)) : 0;
if (sub) {
for (size_t j = 0; j < i; ++j) out[j] = value(j);
} else {
std::fill_n(out, i, 0);
}
for (; i + 8 <= count && top - std::ptrdiff_t(i) >= 7; i += 8) {
const Vec x = reversed(load_any(a.data() + (top - std::ptrdiff_t(i) - 7)));
store_any(out + i, sub ? difference(x, load_any(sub + i)) : x);
}
for (; i < count; ++i) out[i] = value(i);
}
template<class T>
inline void fold(std::span<T> a, Words out) {
const size_t len = out.size();
std::fill(out.begin(), out.end(), 0);
for (size_t from = 0; from < a.size(); from += len) {
const size_t count = std::min(len, a.size() - from), full = count / 8 * 8;
const T* x = a.data() + from;
for (size_t j = 0; j < full; j += 8) {
store(out.data() + j, reduce(add(load(out.data() + j), load_any(x + j)), kP));
}
for (size_t j = full; j < count; ++j) out[j] = (out[j] + input_word(x[j])) % kP;
}
}
template<class T>
inline void long_division(std::span<const T> f, ConstWords g, Words q,
Words r, Words rest) {
using blocked_ntt::detail::multiply_mod;
const size_t d = g.size() - 1;
if constexpr (std::is_same_v<T, Word>) std::copy(f.begin(), f.end(), rest.begin());
else for (size_t i = 0; i < f.size(); ++i) rest[i] = input_word(f[i]);
const Word lead = blocked_ntt::detail::power(g[d], kP - 2);
for (size_t i = q.size(); i-- > 0;) {
const Word c = multiply_mod(rest[i + d], lead);
q[i] = c;
for (size_t t = 0; t < d; ++t) rest[i + t] = (rest[i + t] + kP - multiply_mod(c, g[t])) % kP;
}
std::copy_n(rest.begin(), r.size(), r.begin());
}
template<class T>
inline void remainder_direct(std::span<const T> f, ConstWords g,
ConstWords q, Words r) {
const size_t d = r.size(), k = q.size();
Vec factor[kDirectRemainder];
for (size_t i = 0; i < k; ++i) factor[i] = _mm256_set1_epi64x(std::int64_t((std::uint64_t(q[i]) << 32) % kP));
const auto scalar = [&](size_t j) {
std::uint64_t sum = 0;
for (size_t i = 0; i <= std::min(j, k - 1); ++i) sum = (sum + std::uint64_t(q[i]) * g[j - i]) % kP;
r[j] = Word((input_word(f[j]) + kP - sum) % kP);
};
size_t j = 0;
for (; j < std::min(d, k); ++j) scalar(j);
for (; j + 8 <= d; j += 8) {
Vec total = load_any(f.data() + j);
for (size_t i = 0; i < k; i += 4) {
Vec even = _mm256_setzero_si256(), odd = _mm256_setzero_si256();
for (size_t t = i; t < std::min(k, i + 4); ++t) {
const Vec x = load_any(g.data() + (j - t));
even = _mm256_add_epi64(even, _mm256_mul_epu32(x, factor[t]));
odd = _mm256_add_epi64(odd, _mm256_mul_epu32(_mm256_srli_epi64(x, 32), factor[t]));
}
total = reduce(_mm256_sub_epi32(add(total, broadcast(2 * kP)), montgomery_reduce(even, odd)), 2 * kP);
}
store_any(r.data() + j, reduce(total, kP));
}
for (; j < d; ++j) scalar(j);
}
template<class T>
inline void remainder_wrap(const Transform& t, std::span<const T> f, ConstWords g,
ConstWords q, Words r, Words a,
Words b) {
const size_t len = a.size();
if (q.size() <= len) {
t.forward(q, 0, a);
} else {
fold(q, a);
t.forward(a);
}
ConstWords gl = g;
if (g.size() > len) {
std::copy_n(g.begin(), len, b.begin());
b[0] = (b[0] + g[len]) % kP;
gl = b;
}
t.cyclic_product(gl, 0, b, a);
fold(f, a);
subtract(a.data(), b.data(), r.data(), r.size());
}
template<class T>
inline void remainder_split(const Transform& t, std::span<const T> f, ConstWords g,
ConstWords q, Words r, Words a,
Words b, Words c) {
const size_t len = a.size(), k = q.size(), e = r.size() - len;
t.forward(q, 0, a);
t.cyclic_product(g.subspan(len + 1 - k, k + e - 1), 0, c, a);
subtract(f.data() + len, c.data() + (k - 1), r.data() + len, e);
fold(g, b);
t.cyclic_product(b, a);
fold(f, c);
subtract(c.data(), b.data(), r.data(), len);
subtract(r.data(), r.data() + len, r.data(), e);
}
template <size_t K>
void residual_sum(const Transform& t, std::span<const Transform::Pair> pairs, Words out) {
InverseProductSumBottom<K> bottom{};
for (size_t i = 0; i < K; ++i)
bottom.terms[i] = {{t.roots(), t.inverse_roots(), pairs[i].b.data()}, pairs[i].a.data()};
t.inverse_with(bottom, out, Half::kUpper);
}
inline void residual_sum(const Transform& t, std::span<const Transform::Pair> pairs, Words out) {
switch (pairs.size()) {
case 1:
case 2:
case 3: return t.inverse_product_sum(pairs, out, Half::kUpper);
case 4: return residual_sum<4>(t, pairs, out);
case 5: return residual_sum<5>(t, pairs, out);
case 6: return residual_sum<6>(t, pairs, out);
case 7: return residual_sum<7>(t, pairs, out);
case 8: return residual_sum<8>(t, pairs, out);
default: std::abort();
}
}
template<class T>
inline void subtract_reversed(const T* a, const Word* b, Word* out, size_t count) {
size_t j = 0;
for (; j + 8 <= count; j += 8) store_any(out + j, difference(load_any(a + j), reversed(load_any(b + count - 8 - j))));
for (; j < count; ++j) out[j] = (input_word(a[j]) + kP - b[count - 1 - j]) % kP;
}
template<class T>
inline void divide_blocks(const Transform& t, const DivisionPlan& plan, std::span<const T> f,
ConstWords g, Words q, Words r,
Words scratch) {
const QuotientPlan& p = plan.quotient;
const size_t k = q.size(), s = p.s, len = 2 * s, d = plan.d;
const std::ptrdiff_t top = std::ptrdiff_t(f.size() - 1 + p.pad);
Bump bump(scratch);
const Words G = bump.take(plan.divisor()), ht = bump.take(len);
std::reverse_copy(g.end() - std::ptrdiff_t(G.size()), g.end(), G.begin());
const size_t precision = p.precision(k);
inverse(t, G.first(std::min(G.size(), precision)), ht.first(precision), bump.rest());
t.forward(ht.first(precision), 0, ht);
const Words qs = bump.take(p.blocks * s + s);
Words gt, mp, work;
std::array<Words, kDivisionTerms> windows{}, ring{};
if (plan.tails()) {
gt = bump.take(p.tail), mp = bump.take(p.tail);
t.forward(G, 0, gt);
}
if (p.windows) {
work = bump.take(len);
for (size_t i = 0; i < p.windows; ++i) {
windows[i] = bump.take(len), ring[i] = bump.take(len);
t.forward(G.subspan(i * s, std::min(len, G.size() - i * s)), 0, windows[i]);
}
}
const auto tail_product = [&](size_t j) {
t.cyclic_product(qs.subspan(j * s - d, d), p.tail - 2 * d, mp, gt, Half::kUpper);
return mp.data() + (p.tail - d);
};
for (size_t j = 0; j < p.blocks; ++j) {
Word* const block = qs.data() + j * s;
const std::ptrdiff_t from = top - std::ptrdiff_t(j * s);
if (j == 0) {
reverse_difference(f, from, block, s);
} else if (p.tails) {
reverse_difference(f, from, block, d, tail_product(j));
reverse_difference(f, from - std::ptrdiff_t(d), block + d, s - d);
} else {
const size_t terms = std::min(j, d / s + 1);
std::array<Transform::Pair, kDivisionTerms> pairs;
for (size_t i = 0; i < terms; ++i) pairs[i] = {windows[i], ring[(j - 1 - i) % p.windows]};
residual_sum(t, std::span<const Transform::Pair>(pairs.data(), terms), work);
reverse_difference(f, from, block, s, work.data() + s);
}
t.cyclic_product(qs.subspan(j * s, s), 0, qs.subspan(j * s, len), ht, Half::kLower);
if (p.windows && j + 1 < p.blocks) t.forward(qs.subspan(j * s, s), 0, ring[j % p.windows]);
reverse_difference(qs.first((j + 1) * s), std::ptrdiff_t((j + 1) * s - 1), q.data() + (p.blocks - 1 - j) * s,
j == 0 ? s - p.pad : s);
}
if (plan.remainder.method == Remainder::kTail) subtract_reversed(f.data(), tail_product(p.blocks), r.data(), d);
}
template<class T>
inline void divide(Arena& arena, const DivisionPlan& plan, std::span<const T> f, ConstWords g,
Words q, Words r) {
if (plan.long_division) return long_division(f, g, q, r, arena.take(f.size()));
const Transform t(arena, plan.lg);
const Words scratch = arena.take(plan.scratch());
divide_blocks(t, plan, f, g, q, r, scratch);
Bump bump(scratch);
const size_t len = plan.remainder.length;
switch (plan.remainder.method) {
case Remainder::kDirect: return remainder_direct(f, g, q, r);
case Remainder::kWrap: return remainder_wrap(t, f, g, q, r, bump.take(len), bump.take(len));
case Remainder::kSplit: return remainder_split(t, f, g, q, r, bump.take(len), bump.take(len), bump.take(len));
case Remainder::kTail: return;
}
}
}
inline size_t divide_words(size_t n, size_t m) { return division_detail::DivisionPlan(n, m).words(); }
inline void divide(Arena& arena, ConstWords f, ConstWords g,
Words q, Words r) {
const division_detail::DivisionPlan plan(f.size(), g.size());
if (plan.k == 0) {
std::copy(f.begin(), f.end(), r.begin());
return;
}
division_detail::divide(arena, plan, f, g, q, r);
}
}
namespace fps_detail::blocked_poly {
inline void native_divide(const mint* a, size_t n, const mint* b, size_t m,
std::vector<mint>& quotient, std::vector<mint>& remainder) {
const division_detail::DivisionPlan plan(n, m);
const size_t k = quotient_size(n, m), d = remainder_size(n, m);
Arena arena(plan.words() + Arena::footprint(m) + Arena::footprint(k) + Arena::footprint(d));
const std::span<const mint> f(a, n);
auto g = arena.take(m), q = arena.take(k), r = arena.take(d);
for (size_t i = 0; i < m; ++i) g[i] = std::min(b[i].v, b[i].v - mint::mod());
division_detail::divide(arena, plan, f, g, q, r);
prepare_fps_output(quotient, k), prepare_fps_output(remainder, d);
division_detail::append_division_words(quotient, q.data(), q.size(), blocked_ntt::detail::kR);
division_detail::append_division_words(remainder, r.data(), r.size());
}
inline void native_div_series(const mint* a, size_t input, const mint* b, size_t divisor,
std::vector<mint>& out, size_t n) {
using namespace division_detail;
const size_t count = std::min(divisor, n), degree = count - 1;
QuotientPlan plan;
for (size_t s = 32;; s *= 2) {
QuotientPlan candidate(n, degree, s);
if (candidate.ns < plan.ns) plan = candidate;
if (s >= n || 4 * s > (size_t(1) << Transform::kMaxLog)) break;
}
const size_t s = plan.s, len = 2 * s;
const bool tails = plan.tails && plan.blocks > 1;
const size_t precision = plan.precision(n), numerator = std::min(input, n);
const int lg = std::countr_zero(std::max(len, tails ? plan.tail : 0));
const size_t later = Arena::footprint(plan.blocks * s + s) +
(tails ? 2 * Arena::footprint(plan.tail) : 0) +
(plan.windows ? (2 * plan.windows + 1) * Arena::footprint(len) : 0);
Arena arena(Transform::words(lg) + Arena::footprint(numerator) + Arena::footprint(count) +
Arena::footprint(len) + Arena::footprint(std::max(inverse_scratch(precision), later)));
const Transform t(arena, lg);
auto f = arena.take(numerator), g = arena.take(count), ht = arena.take(len);
for (size_t i = 0; i < numerator; ++i) f[i] = std::min(a[i].v, a[i].v - mint::mod());
for (size_t i = 0; i < count; ++i) g[i] = std::min(b[i].v, b[i].v - mint::mod());
auto scratch = arena.take(std::max(inverse_scratch(precision), later));
inverse(t, g.first(std::min(count, precision)), ht.first(precision), scratch);
t.forward(ht.first(precision), 0, ht);
Bump bump(scratch);
auto qs = bump.take(plan.blocks * s + s);
Words gt, mp, work;
std::array<Words, kDivisionTerms> windows{}, ring{};
if (tails) {
gt = bump.take(plan.tail), mp = bump.take(plan.tail);
t.forward(g, 0, gt);
}
if (plan.windows) {
work = bump.take(len);
for (size_t i = 0; i < plan.windows; ++i) {
windows[i] = bump.take(len), ring[i] = bump.take(len);
t.forward(g.subspan(i * s, std::min(len, count - i * s)), 0, windows[i]);
}
}
prepare_fps_output(out, n);
for (size_t j = 0; j < plan.blocks; ++j) {
auto block = qs.subspan(j * s, len);
const size_t from = j * s, available = from < numerator ? std::min(s, numerator - from) : 0;
if (available) std::copy_n(f.data() + from, available, block.data());
std::fill(block.begin() + available, block.begin() + s, 0u);
if (j && tails) {
t.cyclic_product(qs.subspan(j * s - degree, degree), plan.tail - 2 * degree, mp, gt, Half::kUpper);
subtract(block.data(), mp.data() + plan.tail - degree, block.data(), degree);
} else if (j) {
const size_t terms = std::min(j, degree / s + 1);
std::array<Transform::Pair, kDivisionTerms> pairs;
for (size_t i = 0; i < terms; ++i) pairs[i] = {windows[i], ring[(j - 1 - i) % plan.windows]};
residual_sum(t, std::span<const Transform::Pair>(pairs.data(), terms), work);
subtract(block.data(), work.data() + s, block.data(), s);
}
t.cyclic_product(block.first(s), 0, block, ht, Half::kLower);
if (plan.windows && j + 1 < plan.blocks) t.forward(block.first(s), 0, ring[j % plan.windows]);
append_division_words(out, block.data(), std::min(s, n - from), blocked_ntt::detail::kR);
}
}
}
#pragma GCC pop_options
#endif
#line 6 "components/polynomial_gcd_blocked.hpp"
#if defined(__linux__) && defined(__x86_64__) && defined(__GNUC__) && !defined(__clang__)
#pragma GCC push_options
_Pragma("GCC target(\"avx2\")")
#pragma GCC optimize("O3")
namespace fps_detail::blocked_poly {
namespace gcd_detail {
using namespace detail;
using division_detail::fold, division_detail::subtract, division_detail::montgomery_reduce, division_detail::kLongDivision;
class Stack {
public:
explicit Stack(Words memory) : base_(memory.data()), size_(memory.size()) {}
size_t mark() const { return top_; }
void release(size_t mark) { top_ = mark; }
Word* take(size_t n) {
const size_t need = Arena::footprint(n);
if (need > size_ - top_) throw std::bad_alloc();
Word* const p = base_ + top_;
top_ += need;
return p;
}
private:
Word* base_;
size_t size_, top_ = 0;
};
inline Word mod_mul(Word x, Word y) { return Word(std::uint64_t(x) * y % kP); }
inline Word mod_sub(Word x, Word y) { return x >= y ? x - y : x + kP - y; }
inline Word mod_negate(Word x) { return x ? kP - x : 0; }
inline Word mod_inverse(Word x) { return blocked_ntt::detail::power(x, kP - 2); }
inline void combine3(const Word* u, const Word* v, Word* out, std::ptrdiff_t from,
std::ptrdiff_t to, Word x, Word y, Word z) {
const Vec xs = broadcast(x), ys = broadcast(y), zs = broadcast(z);
for (std::ptrdiff_t j = from & ~std::ptrdiff_t(7); j < to; j += 8) {
const Vec a = load_unaligned(u + j), b = load_unaligned(v + j - 1), c = load_unaligned(v + j);
const Vec even = _mm256_add_epi64(_mm256_add_epi64(_mm256_mul_epu32(a, xs), _mm256_mul_epu32(b, ys)),
_mm256_mul_epu32(c, zs));
const Vec odd = _mm256_add_epi64(
_mm256_add_epi64(_mm256_mul_epu32(_mm256_srli_epi64(a, 32), xs), _mm256_mul_epu32(_mm256_srli_epi64(b, 32), ys)),
_mm256_mul_epu32(_mm256_srli_epi64(c, 32), zs));
store_unaligned(out + j, reduce(montgomery_reduce(even, odd), kP));
}
}
[[gnu::always_inline]] inline void fill_pair(const Word* roots, const Word* x0, const Word* x1,
size_t g, Window (&window)[2][4]) {
const size_t at = 32 * g;
const Vec a0[4] = {load(x0 + at), load(x0 + at + 8), load(x0 + at + 16), load(x0 + at + 24)};
const Vec a1[4] = {load(x1 + at), load(x1 + at + 8), load(x1 + at + 16), load(x1 + at + 24)};
fill_windows(window[0], a0, roots, g);
fill_windows(window[1], a1, roots, g);
}
template <size_t K>
inline void products(const Word* roots, const Word* x0, const Word* x1,
const Word* const (&y_in)[K][2], Word* const (&out_in)[K], size_t n) {
const Word* y[K][2];
Word* out[K];
for (size_t k = 0; k < K; ++k) y[k][0] = y_in[k][0], y[k][1] = y_in[k][1], out[k] = out_in[k];
const Factor undo_montgomery(kR);
Window window[2][2][4];
fill_pair(roots, x0, x1, 0, window[0]);
for (size_t g = 0; g < n / 32; ++g) {
if (g + 1 < n / 32) fill_pair(roots, x0, x1, g + 1, window[(g + 1) & 1]);
const Window(&w)[2][4] = window[g & 1];
const size_t at = 32 * g;
#pragma GCC unroll 4
for (size_t t = 0; t < 4; ++t)
for (size_t k = 0; k < K; ++k) {
const Vec s = low(add(leaf_product(w[0][t], y[k][0] + at + 8 * t), leaf_product(w[1][t], y[k][1] + at + 8 * t)));
store(out[k] + at + 8 * t, reduce(times(s, undo_montgomery), kP));
}
}
}
struct Matrix {
Word* entry[4];
size_t size[4];
Word at0[4];
Word* transform[4];
size_t length[4];
size_t room, progress;
};
class HalfGcd {
public:
static constexpr std::ptrdiff_t kDirect = 64;
static int log_length(size_t n_max) {
return std::max(Transform::kMinLog, int(std::bit_width(std::max<size_t>(n_max, 2) - 1)));
}
static size_t scratch_words(size_t n_max) { return 48 * (size_t(1) << log_length(n_max)) + 4096; }
static size_t words(size_t n_max) {
return Transform::words(log_length(n_max)) + Arena::footprint(scratch_words(n_max));
}
HalfGcd(Arena& arena, size_t n_max, std::ptrdiff_t direct = kDirect)
: transform_(arena, log_length(n_max)), stack_(arena.take(scratch_words(n_max))), direct_(direct) {}
Matrix matrix(size_t k, size_t room) {
Matrix r;
for (int i = 0; i < 4; ++i) {
r.entry[i] = take(k + 1).data();
r.transform[i] = room ? take(room).data() : nullptr;
r.length[i] = 0;
}
r.room = room;
return r;
}
void jump(const Word* a, std::ptrdiff_t n, const Word* b, std::ptrdiff_t m, std::ptrdiff_t k,
Matrix& out, unsigned need = 0xF) {
const std::ptrdiff_t target = n - k;
if (m < target) return identity(out);
if (n > 2 * k) {
const std::ptrdiff_t s = n - 2 * k;
return jump(a + s, n - s, b + s, m - s, k, out, need);
}
if (k <= direct_) return euclid(a, n, b, m, k, out);
const size_t mark = stack_.mark(), room = std::max<size_t>(64, std::bit_ceil(size_t(k)));
const std::ptrdiff_t k1 = first_jump(k), s1 = std::max<std::ptrdiff_t>(0, n - 2 * k1);
Matrix r1 = matrix(size_t(k1), room);
jump(a + s1, n - s1, b + s1, m - s1, k1, r1);
std::ptrdiff_t c_deg = 0;
const Word *c = nullptr, *d = nullptr;
if (r1.progress == 0) {
r1 = matrix(size_t(n - m), room);
Word* const rest = take(size_t(n + 1)).data();
divide_step(a, n, b, m, r1, rest);
c = b, d = rest, c_deg = m;
} else {
c_deg = n - std::ptrdiff_t(r1.progress);
const std::ptrdiff_t s2 = std::max<std::ptrdiff_t>(0, 2 * target - c_deg);
Word *const cs = take(size_t(c_deg + 1)).data(), *const ds = take(size_t(c_deg + 1)).data();
apply(a, n, b, m, r1, c_deg, s2, cs, ds);
c = cs, d = ds;
}
std::ptrdiff_t d_deg = c_deg - 1;
while (d_deg >= target && d[d_deg] == 0) --d_deg;
if (d_deg < target) {
copy(r1, out);
return stack_.release(mark);
}
const std::ptrdiff_t k2 = c_deg - target, s2 = std::max<std::ptrdiff_t>(0, c_deg - 2 * k2);
Matrix r2 = matrix(size_t(k2), room);
jump(c + s2, c_deg - s2, d + s2, d_deg - s2, k2, r2, (need & 3 ? 3u : 0u) | (need & 12 ? 12u : 0u));
combine(r2, r1, out, need);
stack_.release(mark);
}
private:
static std::ptrdiff_t first_jump(std::ptrdiff_t k) {
const size_t u = size_t(k);
return std::ptrdiff_t(std::has_single_bit(u) ? u / 2 : std::bit_floor(u));
}
Words take(size_t n) { return {stack_.take(n), n}; }
Word* take_padded(size_t n) {
Word* const p = stack_.take(n + 16);
std::fill_n(p, n + 16, 0);
return p + 8;
}
static void identity(Matrix& out) {
out.entry[0][0] = out.entry[3][0] = 1;
out.size[0] = out.size[3] = 1;
out.size[1] = out.size[2] = 0;
out.at0[0] = out.at0[3] = 1, out.at0[1] = out.at0[2] = 0;
for (auto& l : out.length) l = 0;
out.progress = 0;
}
static void copy(const Matrix& from, Matrix& to) {
for (int i = 0; i < 4; ++i) {
std::copy_n(from.entry[i], from.size[i], to.entry[i]);
to.size[i] = from.size[i];
to.at0[i] = from.at0[i];
to.length[i] = 0;
}
to.progress = from.progress;
}
void euclid(const Word* a, std::ptrdiff_t n, const Word* b, std::ptrdiff_t m, std::ptrdiff_t k,
Matrix& out) {
const size_t mark = stack_.mark();
const std::ptrdiff_t target = n - k;
std::ptrdiff_t floor = n - 2 * k, lo = std::max<std::ptrdiff_t>(0, floor);
Word* p[3];
for (auto& x : p) x = take_padded(size_t(n + 1));
std::copy(a + lo, a + n + 1, p[0] + lo);
std::copy(b + lo, b + m + 1, p[1] + lo);
const size_t cap = size_t(k) + 8;
Word* row[3][2];
size_t row_size[3][2] = {{1, 0}, {0, 1}, {0, 0}};
for (auto& r : row)
for (auto& e : r) {
e = take_padded(cap);
std::fill_n(e, cap, 0);
}
row[0][0][0] = 1, row[1][1][0] = 1;
Word *c = p[0], *d = p[1], *e = p[2];
std::ptrdiff_t c_deg = n, d_deg = m;
while (d_deg >= target) {
const std::ptrdiff_t delta = c_deg - d_deg;
const Word alpha = d[d_deg];
floor += delta;
lo = std::max<std::ptrdiff_t>(0, floor);
if (delta == 1) {
const Word beta = c[c_deg];
const Word gamma = mod_sub(mod_mul(alpha, c[c_deg - 1]), mod_mul(beta, d[d_deg - 1]));
const Word alpha_r = mod_mul(alpha, kR);
const Word x = mod_mul(alpha, alpha_r), y = kP - mod_mul(beta, alpha_r), z = mod_negate(mod_mul(gamma, kR));
combine3(c, d, e, lo, d_deg, x, y, z);
for (int i = 0; i < 2; ++i) {
const size_t sc = row_size[0][i], sd = row_size[1][i];
const size_t se = std::max(sc, sd ? sd + 1 : 0);
combine3(row[0][i], row[1][i], row[2][i], 0, std::ptrdiff_t(se), x, y, z);
if (row_size[2][i] > se) std::fill(row[2][i] + se, row[2][i] + row_size[2][i], 0);
row_size[2][i] = se;
}
} else {
pseudo_divide(c, c_deg, d, d_deg, e, lo, alpha, row, row_size);
}
Word* const old = c;
c = d, d = e, e = old;
for (int i = 0; i < 2; ++i) {
std::swap(row[0][i], row[1][i]);
std::swap(row[1][i], row[2][i]);
std::swap(row_size[0][i], row_size[1][i]);
std::swap(row_size[1][i], row_size[2][i]);
}
c_deg = d_deg;
d_deg = c_deg - 1;
while (d_deg >= target && d[d_deg] == 0) --d_deg;
}
for (int i = 0; i < 2; ++i)
for (int j = 0; j < 2; ++j) {
std::copy_n(row[i][j], row_size[i][j], out.entry[2 * i + j]);
out.size[2 * i + j] = row_size[i][j];
out.at0[2 * i + j] = row_size[i][j] ? row[i][j][0] : 0;
out.length[2 * i + j] = 0;
}
out.progress = size_t(n - c_deg);
stack_.release(mark);
}
static void pseudo_divide(const Word* c, std::ptrdiff_t c_deg, const Word* d, std::ptrdiff_t d_deg,
Word* e, std::ptrdiff_t lo, Word alpha, Word* (&row)[3][2],
size_t (&row_size)[3][2]) {
std::copy(c + lo, c + c_deg + 1, e + lo);
for (int i = 0; i < 2; ++i) {
std::copy_n(row[0][i], row_size[0][i], row[2][i]);
if (row_size[2][i] > row_size[0][i]) std::fill(row[2][i] + row_size[0][i], row[2][i] + row_size[2][i], 0);
row_size[2][i] = row_size[0][i];
}
for (std::ptrdiff_t t = c_deg - d_deg; t >= 0; --t) {
const Word minus_tau = mod_negate(e[d_deg + t]);
for (std::ptrdiff_t j = lo; j <= d_deg + t; ++j) {
const Word dj = j - t >= 0 ? d[j - t] : 0;
e[j] = Word((std::uint64_t(alpha) * e[j] + std::uint64_t(minus_tau) * dj) % kP);
}
for (int i = 0; i < 2; ++i) {
Word* const re = row[2][i];
const Word* const rd = row[1][i];
const size_t sd = row_size[1][i], shift = size_t(t);
const size_t se = std::max(row_size[2][i], sd ? sd + shift : 0);
for (size_t j = 0; j < se; ++j) {
const Word dj = j >= shift ? rd[j - shift] : 0;
re[j] = Word((std::uint64_t(alpha) * re[j] + std::uint64_t(minus_tau) * dj) % kP);
}
row_size[2][i] = se;
}
}
}
void divide_step(const Word* a, std::ptrdiff_t n, const Word* b, std::ptrdiff_t m, Matrix& r,
Word* rest) {
const size_t delta = size_t(n - m);
Word* q = r.entry[3];
if ((delta + 1) * size_t(m + 1) <= kLongDivision) {
std::copy(a, a + n + 1, rest);
const Word lead = mod_inverse(b[m]);
for (size_t i = delta + 1; i-- > 0;) {
const Word qi = mod_mul(rest[size_t(m) + i], lead);
q[i] = qi;
for (std::ptrdiff_t t = 0; t <= m; ++t)
rest[i + size_t(t)] = mod_sub(rest[i + size_t(t)], mod_mul(qi, b[t]));
}
} else {
const ConstWords f(a, size_t(n + 1)), g(b, size_t(m + 1));
Arena arena(divide_words(f.size(), g.size()));
divide(arena, f, g, Words(q, delta + 1), Words(rest, size_t(m)));
std::fill(rest + m, rest + n + 1, 0);
}
for (size_t i = 0; i <= delta; ++i) q[i] = mod_negate(q[i]);
r.entry[1][0] = r.entry[2][0] = 1;
r.size[0] = 0, r.size[1] = r.size[2] = 1, r.size[3] = delta + 1;
r.at0[0] = 0, r.at0[1] = r.at0[2] = 1, r.at0[3] = q[0];
for (auto& l : r.length) l = 0;
r.progress = delta;
}
void transform_of(const Word* c, size_t size, Words out) const {
const size_t len = out.size();
if (size <= len) return transform_.forward(ConstWords(c, size), 0, out);
fold(ConstWords(c, size), out);
transform_.forward(out);
}
void extend(Matrix& r, int i, size_t len) const {
if (r.length[i] == len) return;
const Words t(r.transform[i], len);
const size_t half = len / 2, size = r.size[i];
if (r.length[i] == half && size <= half + 1) {
const Words upper = t.subspan(half);
if (size <= half) {
transform_.forward_upper(ConstWords(r.entry[i], size), 0, upper);
} else {
std::copy_n(r.entry[i], half, upper.data());
upper[0] = mod_sub(upper[0], r.entry[i][half]);
transform_.forward_upper(upper, 0, upper);
}
} else {
transform_of(r.entry[i], size, t);
}
r.length[i] = len;
}
void apply(const Word* a, std::ptrdiff_t n, const Word* b, std::ptrdiff_t m, Matrix& r1,
std::ptrdiff_t c_deg, std::ptrdiff_t s2, Word* c, Word* d) {
const size_t mark = stack_.mark();
const size_t len = std::max<size_t>(64, std::bit_ceil(size_t(n + 1) / 2));
for (int i = 0; i < 4; ++i) extend(r1, i, len);
const Words ta = take(len), tb = take(len);
transform_of(a, size_t(n + 1), ta);
transform_of(b, size_t(m + 1), tb);
const Words pc = take(len), pd = take(len);
apply_products(r1, ta, tb, pc, pd);
const std::ptrdiff_t l = std::ptrdiff_t(len), mixed = c_deg - l;
if (mixed < 0) {
std::copy(pc.begin() + s2, pc.begin() + c_deg + 1, c + s2);
std::copy(pd.begin() + s2, pd.begin() + c_deg + 1, d + s2);
return stack_.release(mark);
}
const size_t low = size_t(mixed + 1);
const Words lc = take(len), ld = take(len);
const size_t short_len = std::max<size_t>(64, std::bit_ceil(2 * low - 1));
if (short_len < len) {
Matrix r = r1;
for (int i = 0; i < 4; ++i) {
r.transform[i] = take(short_len).data();
transform_.forward(ConstWords(r1.entry[i], std::min(low, r1.size[i])), 0,
Words(r.transform[i], short_len));
}
const Words sa = ta.first(short_len), sb = tb.first(short_len);
transform_.forward(ConstWords(a, low), 0, sa);
transform_.forward(ConstWords(b, std::min(low, size_t(m + 1))), 0, sb);
apply_products(r, sa, sb, lc.first(short_len), ld.first(short_len));
} else {
transform_.forward(ConstWords(a, low), 0, ta);
transform_.forward(ConstWords(b, std::min(low, size_t(m + 1))), 0, tb);
apply_products(r1, ta, tb, lc, ld);
lc[0] = Word((std::uint64_t(r1.at0[0]) * a[0] + std::uint64_t(r1.at0[1]) * b[0]) % kP);
ld[0] = Word((std::uint64_t(r1.at0[2]) * a[0] + std::uint64_t(r1.at0[3]) * b[0]) % kP);
}
const std::ptrdiff_t first = std::min(s2, mixed + 1);
std::copy(lc.begin() + first, lc.begin() + mixed + 1, c + first);
std::copy(ld.begin() + first, ld.begin() + mixed + 1, d + first);
const std::ptrdiff_t from = std::max(s2, mixed + 1);
std::copy(pc.begin() + from, pc.begin() + l, c + from);
std::copy(pd.begin() + from, pd.begin() + l, d + from);
subtract(pc.data(), lc.data(), c + l, size_t(mixed + 1));
subtract(pd.data(), ld.data(), d + l, size_t(mixed + 1));
stack_.release(mark);
}
void apply_products(const Matrix& r1, ConstWords ta, ConstWords tb,
Words c, Words d) const {
const size_t len = c.size();
const auto t = [&](int i) { return ConstWords(r1.transform[i], len); };
const Transform::Pair pc[2] = {{t(0), ta}, {t(1), tb}}, pd[2] = {{t(2), ta}, {t(3), tb}};
transform_.inverse_product_sum(pc, c);
transform_.inverse_product_sum(pd, d);
}
void combine(Matrix& r2, Matrix& r1, Matrix& out, unsigned need) {
const size_t mark = stack_.mark();
const size_t p = r2.progress + r1.progress, len = std::max<size_t>(64, std::bit_ceil(p));
for (int i = 0; i < 2; ++i)
if (need & (3u << 2 * i)) extend(r2, 2 * i, len), extend(r2, 2 * i + 1, len);
for (int j = 0; j < 2; ++j)
if (need & (5u << j)) extend(r1, j, len), extend(r1, 2 + j, len);
const Words buffer = take(len);
const bool keep = out.room >= len;
Word* t[4];
for (int i = 0; i < 4; ++i) t[i] = keep ? out.transform[i] : take(len).data();
for (int i = 0; i < 2; ++i) {
const Word *x0 = r2.transform[2 * i], *x1 = r2.transform[2 * i + 1];
const Word* const y[2][2] = {{r1.transform[0], r1.transform[2]}, {r1.transform[1], r1.transform[3]}};
const unsigned row = need >> 2 * i & 3;
if (row == 3) {
Word* const to[2] = {t[2 * i], t[2 * i + 1]};
products<2>(transform_.roots(), x0, x1, y, to, len);
} else if (row) {
const int j = row == 1 ? 0 : 1;
const Word* const yj[1][2] = {{y[j][0], y[j][1]}};
Word* const to[1] = {t[2 * i + j]};
products<1>(transform_.roots(), x0, x1, yj, to, len);
}
}
for (int i = 0; i < 2; ++i)
for (int j = 0; j < 2; ++j) {
const int ij = 2 * i + j;
out.length[ij] = 0;
const Word x0 = Word(
(std::uint64_t(r2.at0[2 * i]) * r1.at0[j] + std::uint64_t(r2.at0[2 * i + 1]) * r1.at0[2 + j]) % kP);
out.at0[ij] = x0;
if (!(need >> ij & 1)) continue;
if (keep) out.length[ij] = len;
transform_.inverse(ConstWords(t[ij], len), buffer);
Word* const e = out.entry[ij];
std::copy_n(buffer.data(), std::min(p + 1, len), e);
if (p == len) e[len] = mod_sub(e[0], x0), e[0] = x0;
size_t size = p + 1;
while (size > 0 && e[size - 1] == 0) --size;
out.size[ij] = size;
}
out.progress = p;
stack_.release(mark);
}
Transform transform_;
Stack stack_;
std::ptrdiff_t direct_;
};
}
inline size_t inverse_mod_words(size_t n, size_t m) {
const size_t d = m - 1;
return divide_words(n, m) + Arena::footprint(d) + Arena::footprint(quotient_size(n, m)) +
gcd_detail::HalfGcd::words(std::max<size_t>(d, 1));
}
inline std::ptrdiff_t inverse_mod(Arena& arena, ConstWords f, ConstWords g,
Words h) {
using namespace gcd_detail;
const std::ptrdiff_t d = std::ptrdiff_t(g.size()) - 1;
if (d == 0) return 0;
const Words b = arena.take(size_t(d));
const size_t rs = remainder_size(f.size(), g.size());
divide(arena, f, g, arena.take(quotient_size(f.size(), g.size())), b.first(rs));
std::ptrdiff_t m = std::ptrdiff_t(rs) - 1;
while (m >= 0 && b[size_t(m)] == 0) --m;
if (m < 0) return -1;
HalfGcd gcd(arena, size_t(d));
Matrix r = gcd.matrix(size_t(d), 0);
gcd.jump(g.data(), d, b.data(), m, d, r, 0x2);
if (std::ptrdiff_t(r.progress) != d) return -1;
const Word c = Word((std::uint64_t(r.at0[0]) * g[0] + std::uint64_t(r.at0[1]) * b[0]) % kP);
const Word scale = mod_inverse(c);
size_t t = r.size[1];
while (t > 0 && r.entry[1][t - 1] == 0) --t;
for (size_t i = 0; i < t; ++i) h[i] = mod_mul(r.entry[1][i], scale);
return std::ptrdiff_t(t);
}
}
namespace fps_detail::blocked_poly {
inline bool native_inv_mod(const mint* a, size_t n, const mint* b, size_t m, std::vector<mint>& out) {
Arena arena(inverse_mod_words(n, m) + Arena::footprint(n) + Arena::footprint(m) + Arena::footprint(m));
auto f = arena.take(n), g = arena.take(m), h = arena.take(m);
for (size_t i = 0; i < n; ++i) f[i] = std::min(a[i].v, a[i].v - mint::mod());
for (size_t i = 0; i < m; ++i) g[i] = std::min(b[i].v, b[i].v - mint::mod());
const auto size = inverse_mod(arena, f, g, h);
if (size < 0) return false;
prepare_fps_output(out, size_t(size));
division_detail::append_words(out, h.data(), size_t(size), 932051910);
return true;
}
template<class Sink>
inline void native_gcd_matrix(const mint* a, size_t n, const mint* b, size_t m, unsigned need, Sink sink) {
Arena arena(gcd_detail::HalfGcd::words(n - 1) + Arena::footprint(n) + Arena::footprint(m));
auto f = arena.take(n), g = arena.take(m);
for (size_t i = 0; i < n; ++i) f[i] = std::min(a[i].v, a[i].v - mint::mod());
for (size_t i = 0; i < m; ++i) g[i] = std::min(b[i].v, b[i].v - mint::mod());
gcd_detail::HalfGcd gcd(arena, n - 1);
auto matrix = gcd.matrix(n - 1, 0);
gcd.jump(f.data(), std::ptrdiff_t(n) - 1, g.data(), std::ptrdiff_t(m) - 1, std::ptrdiff_t(n) - 1, matrix, need);
sink(matrix);
}
}
#pragma GCC pop_options
#endif