Skip to content
This repository was archived by the owner on Apr 28, 2023. It is now read-only.

Commit 4f90354

Browse files
nicolasvasilacheTheodoros Theodoridis
nicolasvasilache
authored and
Theodoros Theodoridis
committed
Split backend compilation cache
This is the first step in simplifying compilation and reusing caching across backends
1 parent 49bfc2d commit 4f90354

File tree

5 files changed

+235
-179
lines changed

5 files changed

+235
-179
lines changed

‎tc/core/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ add_library(
99

1010
SHARED
1111

12+
compilation_cache.cc
1213
flags.cc
1314
mapping_options.cc
1415
mapping_options_cpp_printer.cc

‎tc/core/compilation_cache.cc

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "tc/core/cuda/cuda_compilation_cache.h"
17+
18+
#include <version.h>
19+
20+
#include <cstdint>
21+
#include <fstream>
22+
#include <numeric>
23+
#include <tuple>
24+
25+
#include "tc/core/cuda/cuda_mapping_options.h"
26+
#include "tc/core/utils/math.h"
27+
28+
namespace tc {
29+
30+
namespace {
31+
uint64_t GetDLTensorAlignment(const DLTensor* t) {
32+
return (reinterpret_cast<std::uintptr_t>(t->data) + t->byte_offset) % 256;
33+
}
34+
} // namespace
35+
36+
detail::TensorInfo::TensorInfo(const DLTensor* t)
37+
: alignment{GetDLTensorAlignment(t)}, dType(t->dtype) {
38+
shape.reserve(t->ndim);
39+
std::copy(t->shape, t->shape + t->ndim, std::back_inserter(shape));
40+
if (not t->strides) {
41+
return;
42+
}
43+
strides.reserve(t->ndim);
44+
std::copy(t->strides, t->strides + t->ndim, std::back_inserter(strides));
45+
}
46+
47+
detail::TensorInfo::TensorInfo(const TensorInfoProto& buf)
48+
: shape{buf.shape().begin(), buf.shape().end()},
49+
strides{buf.strides().begin(), buf.strides().end()},
50+
alignment{buf.alignment()},
51+
dType{static_cast<uint8_t>(buf.dtype().code()),
52+
static_cast<uint8_t>(buf.dtype().bits()),
53+
static_cast<uint16_t>(buf.dtype().lanes())} {}
54+
55+
TensorInfoProto detail::TensorInfo::toProtobuf() const {
56+
TensorInfoProto buf;
57+
buf.mutable_shape()->Reserve(shape.size());
58+
std::copy(
59+
shape.begin(),
60+
shape.end(),
61+
google::protobuf::RepeatedFieldBackInserter(buf.mutable_shape()));
62+
buf.mutable_strides()->Reserve(strides.size());
63+
std::copy(
64+
strides.begin(),
65+
strides.end(),
66+
google::protobuf::RepeatedFieldBackInserter(buf.mutable_strides()));
67+
buf.set_alignment(alignment);
68+
buf.mutable_dtype()->set_code(dType.code);
69+
buf.mutable_dtype()->set_bits(dType.bits);
70+
buf.mutable_dtype()->set_lanes(dType.lanes);
71+
return buf;
72+
}
73+
74+
bool detail::TensorInfo::operator==(const DLTensor* t) const {
75+
if (t->ndim != static_cast<int>(shape.size())) {
76+
return false;
77+
}
78+
79+
auto res = std::mismatch(shape.begin(), shape.end(), t->shape);
80+
if (res.first != shape.end() || res.second != t->shape + t->ndim) {
81+
return false;
82+
}
83+
84+
if (t->strides == nullptr) {
85+
if (strides.size() > 0) {
86+
return false;
87+
}
88+
} else {
89+
if (t->ndim != static_cast<int>(strides.size())) {
90+
return false;
91+
}
92+
93+
res = std::mismatch(strides.begin(), strides.end(), t->strides);
94+
if (res.first != strides.end() || res.second != t->strides + t->ndim) {
95+
return false;
96+
}
97+
}
98+
99+
/*This should be enabled when/if tc starts using alignment information
100+
*if (GetDLTensorAlignment(t) != alignment) {
101+
* return false;
102+
*}
103+
*/
104+
return std::tie(t->dtype.code, t->dtype.bits, t->dtype.lanes) ==
105+
std::tie(dType.code, dType.bits, dType.lanes);
106+
}
107+
108+
bool operator==(const DLDataType& a, const DLDataType& b) {
109+
return a.code == b.code and a.bits == b.bits and a.lanes == b.lanes;
110+
}
111+
112+
bool operator<(const DLDataType& a, const DLDataType& b) {
113+
return a.code < b.code and a.bits < b.bits and a.lanes < b.lanes;
114+
}
115+
116+
bool detail::TensorInfo::operator==(const TensorInfo& t) const {
117+
return alignment == t.alignment and dType == t.dType and shape == t.shape and
118+
strides == t.strides;
119+
}
120+
121+
bool detail::TensorInfo::operator<(const TensorInfo& t) const {
122+
return alignment < t.alignment and dType < t.dType and shape < t.shape and
123+
strides < t.strides;
124+
}
125+
126+
} // namespace tc

‎tc/core/compilation_cache.h

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#pragma once
17+
18+
#include <cstdint>
19+
#include <memory>
20+
#include <mutex>
21+
#include <stdexcept>
22+
#include <string>
23+
#include <vector>
24+
25+
#include <dlpack/dlpack.h>
26+
27+
#include <compcache.pb.h>
28+
29+
#include "tc/core/cuda/cuda.h"
30+
#include "tc/core/cuda/cuda_mapping_options.h"
31+
#include "tc/core/cuda/cuda_rtc.h"
32+
#include "tc/core/utils/time.h"
33+
34+
namespace tc {
35+
36+
namespace detail {
37+
/**
38+
* TensorInfo wraps the necessary bits of DLTensor that are used as part of the
39+
* CompilationCache's entry keys.
40+
*
41+
* It is serializable to protobuf and stored directly in the cache.
42+
*/
43+
struct TensorInfo {
44+
std::vector<int64_t> shape;
45+
std::vector<int64_t> strides;
46+
uint64_t alignment;
47+
DLDataType dType;
48+
49+
TensorInfo(const DLTensor* t);
50+
TensorInfo(const TensorInfoProto& buf);
51+
52+
bool operator==(const DLTensor* t) const;
53+
bool operator==(const TensorInfo& t) const;
54+
bool operator<(const TensorInfo& t) const;
55+
TensorInfoProto toProtobuf() const;
56+
};
57+
} // namespace detail
58+
59+
template <typename CC>
60+
class Cache {
61+
public:
62+
static void enableCache();
63+
static void disableCache();
64+
static void dumpCacheToProtobuf(const std::string& filename);
65+
static void loadCacheFromProtobuf(const std::string& filename);
66+
template <typename Protobuf>
67+
static void loadCacheFromProtobuf(const Protobuf& buf);
68+
static std::shared_ptr<CC> getCache();
69+
static bool cacheEnabled();
70+
71+
size_t size() const;
72+
void clear();
73+
74+
mutable int numberAttemptedRetrievals = 0;
75+
mutable int numberSuccessfulRetrievals = 0;
76+
mutable int numberCacheAttemps = 0;
77+
78+
protected:
79+
// XXX:this should be a std or boost shared_mutex
80+
mutable std::mutex mtx_;
81+
};
82+
83+
class CacheEntrySameKeyDifferentValue : public std::invalid_argument {
84+
public:
85+
explicit CacheEntrySameKeyDifferentValue(const std::string& what_arg)
86+
: invalid_argument(what_arg) {}
87+
explicit CacheEntrySameKeyDifferentValue(const char* what_arg)
88+
: invalid_argument(what_arg) {}
89+
};
90+
91+
} // namespace tc

‎tc/core/cuda/cuda_compilation_cache.cc

Lines changed: 3 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -27,102 +27,6 @@
2727

2828
namespace tc {
2929

30-
namespace {
31-
uint64_t GetDLTensorAlignment(const DLTensor* t) {
32-
return (reinterpret_cast<std::uintptr_t>(t->data) + t->byte_offset) % 256;
33-
}
34-
} // namespace
35-
36-
detail::TensorInfo::TensorInfo(const DLTensor* t)
37-
: alignment{GetDLTensorAlignment(t)}, dType(t->dtype) {
38-
shape.reserve(t->ndim);
39-
std::copy(t->shape, t->shape + t->ndim, std::back_inserter(shape));
40-
if (not t->strides) {
41-
return;
42-
}
43-
strides.reserve(t->ndim);
44-
std::copy(t->strides, t->strides + t->ndim, std::back_inserter(strides));
45-
}
46-
47-
detail::TensorInfo::TensorInfo(const TensorInfoProto& buf)
48-
: shape{buf.shape().begin(), buf.shape().end()},
49-
strides{buf.strides().begin(), buf.strides().end()},
50-
alignment{buf.alignment()},
51-
dType{static_cast<uint8_t>(buf.dtype().code()),
52-
static_cast<uint8_t>(buf.dtype().bits()),
53-
static_cast<uint16_t>(buf.dtype().lanes())} {}
54-
55-
TensorInfoProto detail::TensorInfo::toProtobuf() const {
56-
TensorInfoProto buf;
57-
buf.mutable_shape()->Reserve(shape.size());
58-
std::copy(
59-
shape.begin(),
60-
shape.end(),
61-
google::protobuf::RepeatedFieldBackInserter(buf.mutable_shape()));
62-
buf.mutable_strides()->Reserve(strides.size());
63-
std::copy(
64-
strides.begin(),
65-
strides.end(),
66-
google::protobuf::RepeatedFieldBackInserter(buf.mutable_strides()));
67-
buf.set_alignment(alignment);
68-
buf.mutable_dtype()->set_code(dType.code);
69-
buf.mutable_dtype()->set_bits(dType.bits);
70-
buf.mutable_dtype()->set_lanes(dType.lanes);
71-
return buf;
72-
}
73-
74-
bool detail::TensorInfo::operator==(const DLTensor* t) const {
75-
if (t->ndim != static_cast<int>(shape.size())) {
76-
return false;
77-
}
78-
79-
auto res = std::mismatch(shape.begin(), shape.end(), t->shape);
80-
if (res.first != shape.end() || res.second != t->shape + t->ndim) {
81-
return false;
82-
}
83-
84-
if (t->strides == nullptr) {
85-
if (strides.size() > 0) {
86-
return false;
87-
}
88-
} else {
89-
if (t->ndim != static_cast<int>(strides.size())) {
90-
return false;
91-
}
92-
93-
res = std::mismatch(strides.begin(), strides.end(), t->strides);
94-
if (res.first != strides.end() || res.second != t->strides + t->ndim) {
95-
return false;
96-
}
97-
}
98-
99-
/*This should be enabled when/if tc starts using alignment information
100-
*if (GetDLTensorAlignment(t) != alignment) {
101-
* return false;
102-
*}
103-
*/
104-
return std::tie(t->dtype.code, t->dtype.bits, t->dtype.lanes) ==
105-
std::tie(dType.code, dType.bits, dType.lanes);
106-
}
107-
108-
bool operator==(const DLDataType& a, const DLDataType& b) {
109-
return a.code == b.code and a.bits == b.bits and a.lanes == b.lanes;
110-
}
111-
112-
bool operator<(const DLDataType& a, const DLDataType& b) {
113-
return a.code < b.code and a.bits < b.bits and a.lanes < b.lanes;
114-
}
115-
116-
bool detail::TensorInfo::operator==(const TensorInfo& t) const {
117-
return alignment == t.alignment and dType == t.dType and shape == t.shape and
118-
strides == t.strides;
119-
}
120-
121-
bool detail::TensorInfo::operator<(const TensorInfo& t) const {
122-
return alignment < t.alignment and dType < t.dType and shape < t.shape and
123-
strides < t.strides;
124-
}
125-
12630
namespace {
12731
std::vector<detail::TensorInfo> DLTensorToTensorInfoVector(
12832
const std::vector<const DLTensor*>& ts) {
@@ -134,9 +38,6 @@ std::vector<detail::TensorInfo> DLTensorToTensorInfoVector(
13438
});
13539
return iis;
13640
}
137-
} // namespace
138-
139-
namespace {
14041
std::vector<detail::TensorInfo> ProtoToTensorInfoVector(
14142
const google::protobuf::RepeatedPtrField<TensorInfoProto>& buf) {
14243
std::vector<detail::TensorInfo> iis;
@@ -148,9 +49,6 @@ std::vector<detail::TensorInfo> ProtoToTensorInfoVector(
14849
[](const TensorInfoProto& iip) { return detail::TensorInfo{iip}; });
14950
return iis;
15051
}
151-
} // namespace
152-
153-
namespace {
15452
template <typename Array, typename Buf>
15553
void WriteProtobufArray(const Array& arr, Buf* buf) {
15654
google::protobuf::RepeatedField<typename Array::value_type> data(
@@ -174,21 +72,18 @@ bool operator==(
17472
return true;
17573
}
17674

177-
namespace {
178-
std::shared_ptr<CudaCache> cudaCache_;
179-
std::shared_ptr<OptionsCache> optionsCache_;
180-
std::shared_ptr<ManualCudaCache> manualCudaCache_;
181-
} // namespace
182-
18375
std::shared_ptr<CudaCache>& CudaCache::getGlobalSharedCache() {
76+
static std::shared_ptr<CudaCache> cudaCache_;
18477
return cudaCache_;
18578
}
18679

18780
std::shared_ptr<OptionsCache>& OptionsCache::getGlobalSharedCache() {
81+
static std::shared_ptr<OptionsCache> optionsCache_;
18882
return optionsCache_;
18983
}
19084

19185
std::shared_ptr<ManualCudaCache>& ManualCudaCache::getGlobalSharedCache() {
86+
static std::shared_ptr<ManualCudaCache> manualCudaCache_;
19287
return manualCudaCache_;
19388
}
19489

0 commit comments

Comments
 (0)