Skip to content

Commit 724fdb1

Browse files
Maxon081102Maxon081102
committed
GRA-191: implement pool and crossEntropyLoss
1 parent 928c049 commit 724fdb1

8 files changed

Lines changed: 460 additions & 88 deletions

File tree

‎server/core/Layer.cpp‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include <unordered_map>
22
#include <functional>
3+
#include <cassert>
34
#include <string>
45

56
#include "Layer.h"
@@ -113,4 +114,40 @@ LayerNorm::LayerNorm(const AxisParameters& params,
113114
pipeline.push_back(std::move(_fill));
114115

115116
result = Tensor(div, {pipeline[2], pipeline[5]});
116-
}
117+
}
118+
119+
SoftMax::SoftMax(const AxisParameters& params,
120+
const std::vector<TensorRef>& args)
121+
: sum(params.axis) {
122+
assert(args.size() == 1);
123+
pipeline.reserve(3);
124+
TensorRef tensor = args[0];
125+
126+
Tensor exp_(exp, {tensor});
127+
pipeline.push_back(std::move(exp_));
128+
129+
Tensor sum_(sum, {pipeline[0]});
130+
pipeline.push_back(std::move(sum_));
131+
132+
Tensor fill_(fill, {pipeline[0], pipeline[1]});
133+
pipeline.push_back(std::move(fill_));
134+
135+
result = Tensor(div, {pipeline[0], pipeline[2]});
136+
}
137+
138+
EntropyLoss::EntropyLoss(const CrossEntropyLossParameters& params,
139+
const std::vector<TensorRef>& args)
140+
: softmax({{3}}, {args[0]}), mean({0, 1, 2, 3}), entropy(params.classCount) {
141+
assert(args.size() == 2);
142+
pipeline.reserve(1);
143+
144+
Tensor entropy_(entropy, {softmax.result.value(), args[1]});
145+
pipeline.push_back(std::move(entropy_));
146+
147+
result = Tensor(mean, {pipeline[0]});
148+
}
149+
150+
MaxPool::MaxPool(const std::vector<TensorRef>& args) {
151+
assert(args.size() == 1);
152+
result = Tensor(maxPool, {args[0]});
153+
}

‎server/core/Layer.h‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,28 @@ class LayerNorm: public Layer {
8787
LayerNorm(const AxisParameters& params,
8888
const std::vector<TensorRef>& args);
8989
};
90+
91+
class SoftMax: public Layer {
92+
public:
93+
Exp exp;
94+
SumAxis sum;
95+
Fill fill;
96+
Divide div;
97+
SoftMax(const AxisParameters& params,
98+
const std::vector<TensorRef>& args);
99+
};
100+
101+
class EntropyLoss: public Layer {
102+
public:
103+
SoftMax softmax;
104+
Mean mean;
105+
Entropy entropy;
106+
EntropyLoss(const CrossEntropyLossParameters& params,
107+
const std::vector<TensorRef>& args);
108+
};
109+
110+
class MaxPool: public Layer {
111+
public:
112+
MaxPoolOp maxPool;
113+
MaxPool(const std::vector<TensorRef>& args);
114+
};

‎server/core/LazyBlob.cpp‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,17 @@
11
#include <cassert>
22
#include <vector>
33
#include <optional>
4+
#include <algorithm>
5+
#include <cmath>
6+
47

58
#include "LazyBlob.h"
69
#include "Iterations.h"
710
#include "Allocator.h"
811
#include "Blob.h"
912

1013
#define MAX_DIMS_COUNT 4
14+
#define EPS 1e-9
1115

1216
const Shape& LazyBlob::shape() const {
1317
if (shape_.has_value()) {
@@ -568,3 +572,145 @@ std::ostream& operator<<(std::ostream& os, const LazyBlob &b) {
568572
}
569573
return os;
570574
}
575+
576+
class LazyBlobEntropy: public LazyBlob {
577+
public:
578+
const LazyBlob &a, &b;
579+
const int classCount;
580+
LazyBlobEntropy(const LazyBlob &a, const LazyBlob &b, int classCount):
581+
a(a), b(b), classCount(classCount) {};
582+
583+
void initShape() const final override {
584+
shape_ = b.shape();
585+
}
586+
587+
float operator() (std::size_t k, std::size_t l, std::size_t i, std::size_t j) const override {
588+
assert(b(k, l, i, j) < classCount);
589+
// WARNING: если проблемы, меняем на случай с EPS
590+
return std::log(a(k, l, i, (int) b(k, l, i, j)));
591+
// return std::log(a(k, l, i, (int) b(k, l, i, j)) + EPS);
592+
}
593+
};
594+
595+
class LazyBlobEntropyDerivative: public LazyBlob {
596+
public:
597+
const LazyBlob &a, &b;
598+
const int classCount;
599+
LazyBlobEntropyDerivative(const LazyBlob &a, const LazyBlob &b, int classCount):
600+
a(a), b(b), classCount(classCount) {};
601+
602+
void initShape() const final override {
603+
shape_ = a.shape();
604+
}
605+
606+
float operator() (std::size_t k, std::size_t l, std::size_t i, std::size_t j) const override {
607+
assert(b(k, l, i, j) < classCount);
608+
if (j != (int) b(k, 0, 0, 0)) {
609+
return 0;
610+
}
611+
// WARNING: если проблемы, меняем на случай с EPS
612+
return - 1.0f / (a(k, l, i, j));
613+
// return - 1.0f / (a(k, l, i, j) + EPS);
614+
}
615+
};
616+
617+
const LazyBlob& LazyBlob::entropy(const LazyBlob& a, int classCount) const {
618+
assert(shape().cols() == classCount);
619+
assert(shape().dim4() == a.shape().dim4());
620+
assert(shape().dim3() == 1);
621+
assert(shape().rows() == 1);
622+
assert(a.shape().dim3() == 1);
623+
assert(a.shape().rows() == 1);
624+
assert(a.shape().cols() == 1);
625+
626+
void* location = Allocator::allocateBytes(sizeof(LazyBlobEntropy));
627+
return *(new(location) LazyBlobEntropy(*this, a, classCount));
628+
}
629+
630+
const LazyBlob& LazyBlob::entropyDerivative(const LazyBlob& a, int classCount) const {
631+
assert(shape().cols() == classCount);
632+
assert(shape().dim4() == a.shape().dim4());
633+
assert(shape().dim3() == 1);
634+
assert(shape().rows() == 1);
635+
assert(a.shape().dim3() == 1);
636+
assert(a.shape().rows() == 1);
637+
assert(a.shape().cols() == 1);
638+
void* location = Allocator::allocateBytes(sizeof(LazyBlobEntropyDerivative));
639+
return *(new(location) LazyBlobEntropyDerivative(*this, a, classCount));
640+
}
641+
642+
class LazyBlobMaxPool: public LazyBlob {
643+
public:
644+
const LazyBlob &a;
645+
LazyBlobMaxPool(const LazyBlob &a): a(a) {};
646+
647+
void initShape() const final override {
648+
shape_ = {
649+
{
650+
a.shape().dim4(), a.shape().dim3(), a.shape().rows() / 2, a.shape().cols() / 2
651+
},
652+
a.shape().dimsCount
653+
};
654+
}
655+
656+
float operator() (std::size_t k, std::size_t l, std::size_t i, std::size_t j) const override {
657+
return std::max(
658+
std::max(a(k, l, i * 2, j * 2), a(k, l, i * 2 + 1, j * 2)),
659+
std::max(a(k, l, i * 2, j * 2 + 1), a(k, l, i * 2 + 1, j * 2 + 1))
660+
);
661+
}
662+
};
663+
664+
class LazyBlobMaxPoolDerivative: public LazyBlob {
665+
public:
666+
const LazyBlob &a, &b;
667+
LazyBlobMaxPoolDerivative(const LazyBlob &a, const LazyBlob& b): a(a), b(b) {};
668+
669+
void initShape() const final override {
670+
shape_ = a.shape();
671+
}
672+
673+
float operator() (std::size_t k, std::size_t l, std::size_t i, std::size_t j) const override {
674+
size_t start_i = (i / 2) * 2;
675+
size_t start_j = (j / 2) * 2;
676+
size_t indexOfMax_i = start_i;
677+
size_t indexOfMax_j = start_j;
678+
float max = a(k, l, indexOfMax_i, indexOfMax_j);
679+
if (max < a(k, l, start_i, start_j + 1)) {
680+
indexOfMax_i = start_i;
681+
indexOfMax_j = start_j + 1;
682+
max = a(k, l, indexOfMax_i, indexOfMax_j);
683+
}
684+
685+
if (max < a(k, l, start_i + 1, start_j)) {
686+
indexOfMax_i = start_i + 1;
687+
indexOfMax_j = start_j;
688+
max = a(k, l, indexOfMax_i, indexOfMax_j);
689+
}
690+
691+
if (max < a(k, l, start_i + 1, start_j + 1)) {
692+
indexOfMax_i = start_i + 1;
693+
indexOfMax_j = start_j + 1;
694+
max = a(k, l, indexOfMax_i, indexOfMax_j);
695+
}
696+
697+
if (indexOfMax_i == i && indexOfMax_j == j)
698+
return b(k, l, i / 2, j / 2);
699+
700+
return 0.0f;
701+
}
702+
};
703+
704+
const LazyBlob& LazyBlob::maxPool() const {
705+
assert(shape().cols() % 2 == 0);
706+
assert(shape().rows() % 2 == 0);
707+
void* location = Allocator::allocateBytes(sizeof(LazyBlobEntropyDerivative));
708+
return *(new(location) LazyBlobMaxPool(*this));
709+
}
710+
711+
const LazyBlob& LazyBlob::maxPoolDerivative(const LazyBlob& b) const {
712+
assert(shape().cols() % 2 == 0);
713+
assert(shape().rows() % 2 == 0);
714+
void* location = Allocator::allocateBytes(sizeof(LazyBlobMaxPoolDerivative));
715+
return *(new(location) LazyBlobMaxPoolDerivative(*this, b));
716+
}

‎server/core/LazyBlob.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ class LazyBlob {
3131
const LazyBlob& mean(std::vector<short> axis, bool minusOne = false) const;
3232
const LazyBlob& reverseLast2Dims() const;
3333
const LazyBlob& transposeFirst2Dims() const;
34+
const LazyBlob& entropy(const LazyBlob& b, int classCount) const;
35+
const LazyBlob& entropyDerivative(const LazyBlob& b, int classCount) const;
36+
const LazyBlob& maxPool() const;
37+
const LazyBlob& maxPoolDerivative(const LazyBlob& b) const;
3438

3539
/// To repeat some dimensions several times
3640
/// - Parameter shape: the size we want to get

‎server/core/Operation.cpp‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,3 +273,49 @@ Shape EPS::computeDim(const vector<LazyBlobRef>& args) const {
273273
args1(a);
274274
return a.shape();
275275
}
276+
277+
Blob Exp::compute(const vector<LazyBlobRef>& args) const {
278+
args1(a);
279+
return a.applying([](float x) { return std::exp(x); });
280+
}
281+
282+
vector<LazyBlobRef> Exp::grad(const Blob& grad, const vector<LazyBlobRef>& args) const {
283+
args1(a);
284+
return {grad * a.applying([](float x) { return std::exp(x); })};
285+
}
286+
287+
Shape Exp::computeDim(const vector<LazyBlobRef>& args) const {
288+
args1(a);
289+
return a.shape();
290+
}
291+
292+
Blob Entropy::compute(const vector<LazyBlobRef>& args) const {
293+
args2(a, b);
294+
return -a.entropy(b, classCount);
295+
}
296+
297+
vector<LazyBlobRef> Entropy::grad(const Blob& grad, const vector<LazyBlobRef>& args) const {
298+
args2(a, b);
299+
return {grad.lazy().fill(a.shape()) * a.entropyDerivative(b, classCount), zeroBlob(b.shape())};
300+
}
301+
302+
Shape Entropy::computeDim(const vector<LazyBlobRef>& args) const {
303+
args2(a, b);
304+
(void)a;
305+
return b.shape();
306+
}
307+
308+
Blob MaxPoolOp::compute(const vector<LazyBlobRef>& args) const {
309+
args1(a);
310+
return a.maxPool();
311+
}
312+
313+
vector<LazyBlobRef> MaxPoolOp::grad(const Blob& grad, const vector<LazyBlobRef>& args) const {
314+
args1(a);
315+
return {a.maxPoolDerivative(grad.lazy())};
316+
}
317+
318+
Shape MaxPoolOp::computeDim(const vector<LazyBlobRef>& args) const {
319+
args1(a);
320+
return {{a.shape().dim4(), a.shape().dim3(), a.shape().rows() / 2, a.shape().cols() / 2}, a.shape().dimsCount};
321+
}

‎server/core/Operation.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,3 +131,24 @@ struct EPS: Operation {
131131
std::vector<LazyBlobRef> grad(const Blob& gradient, const std::vector<LazyBlobRef>& args) const override;
132132
Shape computeDim(const std::vector<LazyBlobRef>& args) const override;
133133
};
134+
135+
struct Exp: Operation {
136+
std::string name = "Exp";
137+
Blob compute(const std::vector<LazyBlobRef>& args) const override;
138+
std::vector<LazyBlobRef> grad(const Blob& gradient, const std::vector<LazyBlobRef>& args) const override;
139+
Shape computeDim(const std::vector<LazyBlobRef>& args) const override;
140+
};
141+
142+
struct Entropy: Operation {
143+
int classCount;
144+
Entropy(int classCouont): classCount(classCouont) {};
145+
Blob compute(const std::vector<LazyBlobRef>& args) const override;
146+
std::vector<LazyBlobRef> grad(const Blob& gradient, const std::vector<LazyBlobRef>& args) const override;
147+
Shape computeDim(const std::vector<LazyBlobRef>& args) const override;
148+
};
149+
150+
struct MaxPoolOp: Operation {
151+
Blob compute(const std::vector<LazyBlobRef>& args) const override;
152+
std::vector<LazyBlobRef> grad(const Blob& gradient, const std::vector<LazyBlobRef>& args) const override;
153+
Shape computeDim(const std::vector<LazyBlobRef>& args) const override;
154+
};

‎server/core/Parameters.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,7 @@ struct AxisParameters
2424
std::vector<short> axis;
2525
};
2626

27+
struct CrossEntropyLossParameters
28+
{
29+
std::size_t classCount;
30+
};

0 commit comments

Comments
 (0)