Skip to content

Instantly share code, notes, and snippets.

@eiennohito
Created June 13, 2018 03:01
Show Gist options
  • Save eiennohito/5822be4a80a4fdbd88a569e6c5f10d4f to your computer and use it in GitHub Desktop.
Save eiennohito/5822be4a80a4fdbd88a569e6c5f10d4f to your computer and use it in GitHub Desktop.
#include "core/impl/feature_impl_combine.h"
#include "core/impl/feature_impl_ngram_partial.h"
#include "core/impl/feature_impl_prim.h"
#include "core/impl/feature_impl_ngram_partial_kernels.h"
#include "t9_mini.cg.h"
namespace jumanpp_generated {
namespace {
class NgramFeatureStaticApply_JppT9Mini final : public ::jumanpp::core::features::impl::NgramFeatureApplyImpl < NgramFeatureStaticApply_JppT9Mini > {
public :
inline void apply(jumanpp::util::MutableArraySlice<jumanpp::u32> result,
const jumanpp::util::ArraySlice<jumanpp::u64> &t2,
const jumanpp::util::ArraySlice<jumanpp::u64> &t1,
const jumanpp::util::ArraySlice<jumanpp::u64> &t0) const noexcept {
result.at(0) = static_cast<::jumanpp::u32>(jumanpp::util::hashing::FastHash1{}.mix(3).mix(::jumanpp::u64{0ULL}).mix(::jumanpp::u64{5575843856927ULL}).mix(t0.at(1)).result());
result.at(1) = static_cast<::jumanpp::u32>(jumanpp::util::hashing::FastHash1{}.mix(3).mix(::jumanpp::u64{1ULL}).mix(::jumanpp::u64{5575843856927ULL}).mix(t0.at(0)).result());
result.at(2) = static_cast<::jumanpp::u32>(jumanpp::util::hashing::FastHash1{}.mix(3).mix(::jumanpp::u64{2ULL}).mix(::jumanpp::u64{5575843856927ULL}).mix(t0.at(2)).result());
result.at(3) = static_cast<::jumanpp::u32>(jumanpp::util::hashing::FastHash1{}.mix(4).mix(::jumanpp::u64{3ULL}).mix(::jumanpp::u64{5575847461935ULL}).mix(t0.at(1)).mix(t1.at(1)).result());
result.at(4) = static_cast<::jumanpp::u32>(jumanpp::util::hashing::FastHash1{}.mix(4).mix(::jumanpp::u64{4ULL}).mix(::jumanpp::u64{5575847461935ULL}).mix(t0.at(0)).mix(t1.at(0)).result());
result.at(5) = static_cast<::jumanpp::u32>(jumanpp::util::hashing::FastHash1{}.mix(4).mix(::jumanpp::u64{5ULL}).mix(::jumanpp::u64{5575847461935ULL}).mix(t0.at(2)).mix(t1.at(2)).result());
result.at(6) = static_cast<::jumanpp::u32>(jumanpp::util::hashing::FastHash1{}.mix(5).mix(::jumanpp::u64{6ULL}).mix(::jumanpp::u64{89213361200927ULL}).mix(t0.at(0)).mix(t1.at(0)).mix(t2.at(0)).result());
} // void apply
}; // class NgramFeatureStaticApply_JppT9Mini
class PartNgramFeatureStaticApply_JppT9Mini final : public ::jumanpp::core::features::impl::PartialNgramFeatureApplyImpl< PartNgramFeatureStaticApply_JppT9Mini > {
public:
void uniStep0(jumanpp::util::ArraySlice<jumanpp::u64> patterns, jumanpp::u32 mask, ::jumanpp::core::analysis::WeightBuffer weights, jumanpp::util::MutableArraySlice<jumanpp::u32> result) const noexcept {
constexpr ::jumanpp::core::features::impl::UnigramFeature fng_uni_0_{0, 0, 1};
auto uni_v_0 = fng_uni_0_.step0(patterns, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(uni_v_0);
constexpr ::jumanpp::core::features::impl::UnigramFeature fng_uni_1_{1, 1, 0};
auto uni_v_1 = fng_uni_1_.step0(patterns, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(uni_v_1);
constexpr ::jumanpp::core::features::impl::UnigramFeature fng_uni_2_{2, 2, 2};
auto uni_v_2 = fng_uni_2_.step0(patterns, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(uni_v_2);
}
void biStep0(jumanpp::util::ArraySlice<jumanpp::u64> patterns, jumanpp::util::MutableArraySlice<jumanpp::u64> state) const noexcept {
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_3_{0, 3, 1, 1};
fng_bi_3_.step0(patterns, state);
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_4_{1, 4, 0, 0};
fng_bi_4_.step0(patterns, state);
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_5_{2, 5, 2, 2};
fng_bi_5_.step0(patterns, state);
}
void biStep1(jumanpp::util::ArraySlice<jumanpp::u64> patterns, jumanpp::util::ArraySlice<jumanpp::u64> state, jumanpp::u32 mask, jumanpp::core::analysis::WeightBuffer weights, jumanpp::util::MutableArraySlice<jumanpp::u32> result) const noexcept {
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_3_{0, 3, 1, 1};
auto bi_val_0 = fng_bi_3_.step1(patterns, state, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(bi_val_0);
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_4_{1, 4, 0, 0};
auto bi_val_1 = fng_bi_4_.step1(patterns, state, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(bi_val_1);
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_5_{2, 5, 2, 2};
auto bi_val_2 = fng_bi_5_.step1(patterns, state, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(bi_val_2);
}
void triStep0(jumanpp::util::ArraySlice<jumanpp::u64> patterns, jumanpp::util::MutableArraySlice<jumanpp::u64> state) const noexcept {
constexpr ::jumanpp::core::features::impl::TrigramFeature fng_tri_6_{0, 6, 0, 0, 0};
fng_tri_6_.step0(patterns, state);
}
void triStep1(jumanpp::util::ArraySlice<jumanpp::u64> patterns, jumanpp::util::ArraySlice<jumanpp::u64> state, jumanpp::util::MutableArraySlice<jumanpp::u64> result) const noexcept {
constexpr ::jumanpp::core::features::impl::TrigramFeature fng_tri_6_{0, 6, 0, 0, 0};
fng_tri_6_.step1(patterns, state, result);
}
void triStep2(jumanpp::util::ArraySlice<jumanpp::u64> patterns, jumanpp::util::ArraySlice<jumanpp::u64> state, jumanpp::u32 mask, jumanpp::core::analysis::WeightBuffer weights, jumanpp::util::MutableArraySlice<jumanpp::u32> result) const noexcept {
constexpr ::jumanpp::core::features::impl::TrigramFeature fng_tri_6_{0, 6, 0, 0, 0};
auto tri_v_0 = fng_tri_6_.step2(patterns, state, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(tri_v_0);
}
void biFull(jumanpp::util::ArraySlice<jumanpp::u64> t0, jumanpp::util::ArraySlice<jumanpp::u64> t1, jumanpp::u32 mask, jumanpp::core::analysis::WeightBuffer weights, jumanpp::util::MutableArraySlice<jumanpp::u32> result) const noexcept {
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_3_{0, 3, 1, 1};
auto r_0 = fng_bi_3_.jointApply(t0, t1, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(r_0);
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_4_{1, 4, 0, 0};
auto r_1 = fng_bi_4_.jointApply(t0, t1, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(r_1);
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_5_{2, 5, 2, 2};
auto r_2 = fng_bi_5_.jointApply(t0, t1, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(r_2);
}
void triFull(jumanpp::util::ArraySlice<jumanpp::u64> t0,
jumanpp::util::ArraySlice<jumanpp::u64> t1,
jumanpp::util::ArraySlice<jumanpp::u64> t2,
jumanpp::u32 mask,
jumanpp::core::analysis::WeightBuffer weights,
jumanpp::util::MutableArraySlice<jumanpp::u32> result
) const noexcept {
constexpr ::jumanpp::core::features::impl::TrigramFeature fng_tri_6_{0, 6, 0, 0, 0};
auto r_0 = fng_tri_6_.jointApply(t0, t1, t2, result, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(r_0);
}
void allocateBuffers(::jumanpp::core::features::FeatureBuffer* fbuf,const ::jumanpp::core::features::AnalysisRunStats& stats,::jumanpp::util::memory::PoolAlloc* alloc) const override {
using namespace jumanpp;
u32 maxNgrams = std::max({numUnigrams(), numBigrams(), numTrigrams()});
fbuf->currentElems = ~0u;
fbuf->valueBuffer1 = alloc->allocateBuf<u32>(maxNgrams, 64);
fbuf->valueBuffer2 = alloc->allocateBuf<u32>(maxNgrams, 64);
fbuf->t1Buffer = alloc->allocateBuf<u64>(numBigrams() * stats.maxStarts, 64);
fbuf->t2Buffer1 = alloc->allocateBuf<u64>(numTrigrams() * stats.maxStarts, 64);
fbuf->t2Buffer2 = alloc->allocateBuf<u64>(numTrigrams() * stats.maxStarts, 64);
fbuf->scoreBuffer = alloc->allocateBuf<float>(stats.maxEnds, 16);
}
::jumanpp::u32 numUnigrams() const noexcept override { return 3; }
::jumanpp::u32 numBigrams() const noexcept override { return 3; }
::jumanpp::u32 numTrigrams() const noexcept override { return 1; }
void applyBiStep2(::jumanpp::core::features::FeatureBuffer* buffers, ::jumanpp::util::ArraySlice<jumanpp::u64> p1, ::jumanpp::core::analysis::FeatureScorer* scorer, ::jumanpp::util::MutableArraySlice<float> result) const noexcept override {
auto numElems = buffers->currentElems;
if (numElems == 0) { return; }
auto numBigrams = this->numBigrams();
auto buf1 = buffers->valBuf1(numBigrams);
auto buf2 = buffers->valBuf2(numBigrams);
const auto state = buffers->t1Buf(numBigrams, numElems);
auto weights = scorer->weights();
auto mask = static_cast<::jumanpp::u32>(weights.size() - 1);
this->biStep1(p1, state.row(0), mask, weights, buf2);
::jumanpp::u32 row = 1;
for (; row < state.numRows(); ++row) {
auto srow = state.row(row);
float f_0 = 0;
float f_1 = 0;
{
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_3_{0, 3, 1, 1};
f_0 += weights.at(buf2.at(0));
::jumanpp::u32 idx = fng_bi_3_.step1(p1, srow, buf1, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(idx);
}
{
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_4_{1, 4, 0, 0};
f_1 += weights.at(buf2.at(1));
::jumanpp::u32 idx = fng_bi_4_.step1(p1, srow, buf1, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(idx);
}
{
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_5_{2, 5, 2, 2};
f_0 += weights.at(buf2.at(2));
::jumanpp::u32 idx = fng_bi_5_.step1(p1, srow, buf1, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(idx);
}
result.at(row - 1) += (f_0 + f_1);
buf1.swap(buf2);
}
result.at(row - 1) += ::jumanpp::core::analysis::impl::computeUnrolled4RawPerceptron(weights, buf2);
}
void applyTriStep3(::jumanpp::core::features::FeatureBuffer* buffers, ::jumanpp::util::ArraySlice<jumanpp::u64> p2, ::jumanpp::core::analysis::FeatureScorer* scorer, ::jumanpp::util::MutableArraySlice<float> result) const noexcept override {
auto numElems = buffers->currentElems;
if (numElems == 0) { return; }
auto numTrigrams = this->numTrigrams();
auto buf1 = buffers->valBuf1(numTrigrams);
auto buf2 = buffers->valBuf2(numTrigrams);
const auto state = buffers->t2Buf2(numTrigrams, numElems);
auto weights = scorer->weights();
auto mask = static_cast<::jumanpp::u32>(weights.size() - 1);
this->triStep2(p2, state.row(0), mask, weights, buf2);
::jumanpp::u32 row = 1;
for (; row < state.numRows(); ++row) {
auto srow = state.row(row);
float f_0 = 0;
{
constexpr ::jumanpp::core::features::impl::TrigramFeature fng_tri_6_{0, 6, 0, 0, 0};
f_0 += weights.at(buf2.at(0));
::jumanpp::u32 idx = fng_tri_6_.step2(p2, srow, buf1, mask);
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(idx);
}
result.at(row - 1) += (f_0);
buf1.swap(buf2);
}
result.at(row - 1) += ::jumanpp::core::analysis::impl::computeUnrolled4RawPerceptron(weights, buf2);
}
void applyBiTri(::jumanpp::core::features::FeatureBuffer* buffers,
::jumanpp::u32 t0idx,
::jumanpp::util::ArraySlice<jumanpp::u64> t0,
::jumanpp::util::ConstSliceable<jumanpp::u64> t1,
::jumanpp::util::ConstSliceable<jumanpp::u64> t2,
::jumanpp::util::ArraySlice<jumanpp::u32> t1idxes,
::jumanpp::core::analysis::FeatureScorer* scorer,
::jumanpp::util::MutableArraySlice<float> result) const noexcept override {
constexpr auto numBigrams = 3;
constexpr auto numTrigrams = 1;
auto weights = scorer->weights();
auto scbuf = buffers->scoreBuf(t1.numRows());
const auto bistateBuf = buffers->t1Buf(numBigrams, buffers->currentElems);
const auto tristateBuf = buffers->t2Buf1(numTrigrams, buffers->currentElems);
auto buf1 = buffers->valBuf1(numBigrams);
auto buf2 = buffers->valBuf2(numBigrams);
static constexpr jumanpp::u32 t1BiFeatureIdxes[numBigrams] = {
1, 0, 2,
};
static constexpr jumanpp::u32 t1TriFeatureIdxes[numTrigrams] = {
0,
};
static constexpr jumanpp::u32 t2TriFeatureIdxes[numTrigrams] = {
0,
};
::jumanpp::core::features::impl::applyBiTriFullKernel(
bistateBuf.row(t0idx),
tristateBuf.row(t0idx),
t1,
t2,
t1idxes,
t1BiFeatureIdxes,
t1TriFeatureIdxes,
t2TriFeatureIdxes,
buf1,
buf2,
weights,
scbuf,
result
);
}
}; // class PartNgramFeatureStaticApply_JppT9Mini
class PatternFeatureStaticApply_JppT9Mini : public ::jumanpp::core::features::GeneratedPatternFeatureApply {
void patternsAndUnigramsApply(
::jumanpp::core::features::impl::PrimitiveFeatureContext *ctx,
::jumanpp::util::ArraySlice<::jumanpp::core::NodeInfo> nodeInfos,
::jumanpp::util::Sliceable<::jumanpp::i32> entryFeatureBuffer,
::jumanpp::core::features::FeatureBuffer *fbuffer,
::jumanpp::util::Sliceable<::jumanpp::u64> patternMatrix,
const ::jumanpp::core::analysis::FeatureScorer *scorer,
::jumanpp::util::MutableArraySlice<float> scores) const override {
auto numItems = patternMatrix.numRows();
const auto weights = scorer->weights();
::jumanpp::u32 mask = weights.size() - 1;
auto t1state = fbuffer->t1Buf(3, numItems);
auto t2state = fbuffer->t2Buf1(1, numItems);
auto buf1 = fbuffer->valBuf1(3);
auto buf2 = fbuffer->valBuf2(3);
for (int item = 0; item < numItems; ++item) {
auto patterns = patternMatrix.row(item);
auto& nodeInfo = nodeInfos.at(item);
auto entry = entryFeatureBuffer.row(item);
bool status = ctx->fillEntryBuffer(nodeInfo.entryPtr(), entry);
JPP_DCHECK(status);
auto t1row = t1state.row(item);
auto t2row = t2state.row(item);
// preload memory of next item
if (JPP_LIKELY(item < (numItems - 1))) {
ctx->prefetchDicItem(nodeInfos.at(item + 1).entryPtr());
}
buf1.swap(buf2);
// pattern feature #1 (with unigram), usage=3
constexpr jumanpp::core::features::impl::CopyPrimFeatureImpl pfobj_surface_{0};
::jumanpp::u64 pf_surface_0 = pfobj_surface_.access(ctx, nodeInfo, entry);
auto fe_pat_hash_1 = ::jumanpp::util::hashing::FastHash1{}.mix(1ULL).mix(1ULL).mix(34359656763621376ULL);
fe_pat_hash_1 = fe_pat_hash_1.mix(pf_surface_0);
::jumanpp::u64 fe_pat_1 = fe_pat_hash_1.result();
constexpr ::jumanpp::core::features::impl::UnigramFeature fng_uni_0_{0, 0, 1};
auto value_fng_uni_0_ = fng_uni_0_.maskedValueFor(fe_pat_1, mask);
float score_part_0 = weights.at(buf2.at(0)); // perceptron op
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(value_fng_uni_0_);
buf1.at(0) = value_fng_uni_0_;
patterns.at(1) = fe_pat_1;
// pattern feature #0 (with unigram), usage=7
constexpr jumanpp::core::features::impl::CopyPrimFeatureImpl pfobj_english_{1};
::jumanpp::u64 pf_english_1 = pfobj_english_.access(ctx, nodeInfo, entry);
auto fe_pat_hash_0 = ::jumanpp::util::hashing::FastHash1{}.mix(0ULL).mix(1ULL).mix(34359656763621376ULL);
fe_pat_hash_0 = fe_pat_hash_0.mix(pf_english_1);
::jumanpp::u64 fe_pat_0 = fe_pat_hash_0.result();
constexpr ::jumanpp::core::features::impl::UnigramFeature fng_uni_1_{1, 1, 0};
auto value_fng_uni_1_ = fng_uni_1_.maskedValueFor(fe_pat_0, mask);
score_part_0 += weights.at(buf2.at(1)); // perceptron op
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(value_fng_uni_1_);
buf1.at(1) = value_fng_uni_1_;
patterns.at(0) = fe_pat_0;
// pattern feature #2 (with unigram), usage=3
auto fe_pat_hash_2 = ::jumanpp::util::hashing::FastHash1{}.mix(2ULL).mix(2ULL).mix(34359656763621376ULL);
fe_pat_hash_2 = fe_pat_hash_2.mix(pf_surface_0);
fe_pat_hash_2 = fe_pat_hash_2.mix(pf_english_1);
::jumanpp::u64 fe_pat_2 = fe_pat_hash_2.result();
constexpr ::jumanpp::core::features::impl::UnigramFeature fng_uni_2_{2, 2, 2};
auto value_fng_uni_2_ = fng_uni_2_.maskedValueFor(fe_pat_2, mask);
score_part_0 += weights.at(buf2.at(2)); // perceptron op
weights.prefetch<::jumanpp::util::PrefetchHint::PREFETCH_HINT_T0>(value_fng_uni_2_);
buf1.at(2) = value_fng_uni_2_;
patterns.at(2) = fe_pat_2;
// bigram and trigram state
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_3_{0, 3, 1, 1};
fng_bi_3_.step0(patterns, t1row);
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_4_{1, 4, 0, 0};
fng_bi_4_.step0(patterns, t1row);
constexpr ::jumanpp::core::features::impl::BigramFeature fng_bi_5_{2, 5, 2, 2};
fng_bi_5_.step0(patterns, t1row);
constexpr ::jumanpp::core::features::impl::TrigramFeature fng_tri_6_{0, 6, 0, 0, 0};
fng_tri_6_.step0(patterns, t2row);
// publish perceptron value
if (JPP_LIKELY(item > 0)) {
scores.at(item - 1) = score_part_0;
}
}
scores.at(numItems - 1) = ::jumanpp::core::analysis::impl::computeUnrolled4RawPerceptron(weights, buf1);
} //end function
}; // class PatternFeatureStaticApply_JppT9Mini
} //anon namespace
jumanpp::core::features::NgramFeatureApply*JppT9Mini::ngram() const {
return new NgramFeatureStaticApply_JppT9Mini{};
}
jumanpp::core::features::PartialNgramFeatureApply*JppT9Mini::ngramPartial() const {
return new PartNgramFeatureStaticApply_JppT9Mini{};
}
jumanpp::core::features::GeneratedPatternFeatureApply*JppT9Mini::pattern() const {
return new PatternFeatureStaticApply_JppT9Mini{};
}
} //jumanpp_generated namespace
#include "core/features_api.h"
namespace jumanpp_generated {
class JppT9Mini: public jumanpp::core::features::StaticFeatureFactory {
jumanpp::core::features::NgramFeatureApply*ngram() const override;
jumanpp::core::features::PartialNgramFeatureApply*ngramPartial() const override;
jumanpp::core::features::GeneratedPatternFeatureApply*pattern() const override;
};
} //namespace jumanpp_generated
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment