BinaryStreamArray.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. //===- BinaryStreamArray.h - Array backed by an arbitrary stream *- C++ -*-===//
  2. //
  3. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4. // See https://llvm.org/LICENSE.txt for license information.
  5. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6. //
  7. //===----------------------------------------------------------------------===//
  8. ///
  9. /// \file
  10. /// Lightweight arrays that are backed by an arbitrary BinaryStream. This file
  11. /// provides two different array implementations.
  12. ///
  13. /// VarStreamArray - Arrays of variable length records. The user specifies
  14. /// an Extractor type that can extract a record from a given offset and
  15. /// return the number of bytes consumed by the record.
  16. ///
  17. /// FixedStreamArray - Arrays of fixed length records. This is similar in
  18. /// spirit to ArrayRef<T>, but since it is backed by a BinaryStream, the
  19. /// elements of the array need not be laid out in contiguous memory.
  20. ///
  21. #ifndef LLVM_SUPPORT_BINARYSTREAMARRAY_H
  22. #define LLVM_SUPPORT_BINARYSTREAMARRAY_H
  23. #include "llvm/ADT/ArrayRef.h"
  24. #include "llvm/ADT/iterator.h"
  25. #include "llvm/Support/Alignment.h"
  26. #include "llvm/Support/BinaryStreamRef.h"
  27. #include "llvm/Support/Error.h"
  28. #include <cassert>
  29. #include <cstdint>
  30. namespace llvm {
  31. /// VarStreamArrayExtractor is intended to be specialized to provide customized
  32. /// extraction logic. On input it receives a BinaryStreamRef pointing to the
  33. /// beginning of the next record, but where the length of the record is not yet
  34. /// known. Upon completion, it should return an appropriate Error instance if
  35. /// a record could not be extracted, or if one could be extracted it should
  36. /// return success and set Len to the number of bytes this record occupied in
  37. /// the underlying stream, and it should fill out the fields of the value type
  38. /// Item appropriately to represent the current record.
  39. ///
  40. /// You can specialize this template for your own custom value types to avoid
  41. /// having to specify a second template argument to VarStreamArray (documented
  42. /// below).
  43. template <typename T> struct VarStreamArrayExtractor {
  44. // Method intentionally deleted. You must provide an explicit specialization
  45. // with the following method implemented.
  46. Error operator()(BinaryStreamRef Stream, uint32_t &Len,
  47. T &Item) const = delete;
  48. };
  49. /// VarStreamArray represents an array of variable length records backed by a
  50. /// stream. This could be a contiguous sequence of bytes in memory, it could
  51. /// be a file on disk, or it could be a PDB stream where bytes are stored as
  52. /// discontiguous blocks in a file. Usually it is desirable to treat arrays
  53. /// as contiguous blocks of memory, but doing so with large PDB files, for
  54. /// example, could mean allocating huge amounts of memory just to allow
  55. /// re-ordering of stream data to be contiguous before iterating over it. By
  56. /// abstracting this out, we need not duplicate this memory, and we can
  57. /// iterate over arrays in arbitrarily formatted streams. Elements are parsed
  58. /// lazily on iteration, so there is no upfront cost associated with building
  59. /// or copying a VarStreamArray, no matter how large it may be.
  60. ///
  61. /// You create a VarStreamArray by specifying a ValueType and an Extractor type.
  62. /// If you do not specify an Extractor type, you are expected to specialize
  63. /// VarStreamArrayExtractor<T> for your ValueType.
  64. ///
  65. /// By default an Extractor is default constructed in the class, but in some
  66. /// cases you might find it useful for an Extractor to maintain state across
  67. /// extractions. In this case you can provide your own Extractor through a
  68. /// secondary constructor. The following examples show various ways of
  69. /// creating a VarStreamArray.
  70. ///
  71. /// // Will use VarStreamArrayExtractor<MyType> as the extractor.
  72. /// VarStreamArray<MyType> MyTypeArray;
  73. ///
  74. /// // Will use a default-constructed MyExtractor as the extractor.
  75. /// VarStreamArray<MyType, MyExtractor> MyTypeArray2;
  76. ///
  77. /// // Will use the specific instance of MyExtractor provided.
  78. /// // MyExtractor need not be default-constructible in this case.
  79. /// MyExtractor E(SomeContext);
  80. /// VarStreamArray<MyType, MyExtractor> MyTypeArray3(E);
  81. ///
  82. template <typename ValueType, typename Extractor> class VarStreamArrayIterator;
  83. template <typename ValueType,
  84. typename Extractor = VarStreamArrayExtractor<ValueType>>
  85. class VarStreamArray {
  86. friend class VarStreamArrayIterator<ValueType, Extractor>;
  87. public:
  88. typedef VarStreamArrayIterator<ValueType, Extractor> Iterator;
  89. VarStreamArray() = default;
  90. explicit VarStreamArray(const Extractor &E) : E(E) {}
  91. explicit VarStreamArray(BinaryStreamRef Stream, uint32_t Skew = 0)
  92. : Stream(Stream), Skew(Skew) {}
  93. VarStreamArray(BinaryStreamRef Stream, const Extractor &E, uint32_t Skew = 0)
  94. : Stream(Stream), E(E), Skew(Skew) {}
  95. Iterator begin(bool *HadError = nullptr) const {
  96. return Iterator(*this, E, Skew, nullptr);
  97. }
  98. bool valid() const { return Stream.valid(); }
  99. uint32_t skew() const { return Skew; }
  100. Iterator end() const { return Iterator(E); }
  101. bool empty() const { return Stream.getLength() == 0; }
  102. VarStreamArray<ValueType, Extractor> substream(uint32_t Begin,
  103. uint32_t End) const {
  104. assert(Begin >= Skew);
  105. // We should never cut off the beginning of the stream since it might be
  106. // skewed, meaning the initial bytes are important.
  107. BinaryStreamRef NewStream = Stream.slice(0, End);
  108. return {NewStream, E, Begin};
  109. }
  110. /// given an offset into the array's underlying stream, return an
  111. /// iterator to the record at that offset. This is considered unsafe
  112. /// since the behavior is undefined if \p Offset does not refer to the
  113. /// beginning of a valid record.
  114. Iterator at(uint32_t Offset) const {
  115. return Iterator(*this, E, Offset, nullptr);
  116. }
  117. const Extractor &getExtractor() const { return E; }
  118. Extractor &getExtractor() { return E; }
  119. BinaryStreamRef getUnderlyingStream() const { return Stream; }
  120. void setUnderlyingStream(BinaryStreamRef NewStream, uint32_t NewSkew = 0) {
  121. Stream = NewStream;
  122. Skew = NewSkew;
  123. }
  124. void drop_front() { Skew += begin()->length(); }
  125. private:
  126. BinaryStreamRef Stream;
  127. Extractor E;
  128. uint32_t Skew = 0;
  129. };
  130. template <typename ValueType, typename Extractor>
  131. class VarStreamArrayIterator
  132. : public iterator_facade_base<VarStreamArrayIterator<ValueType, Extractor>,
  133. std::forward_iterator_tag, ValueType> {
  134. typedef VarStreamArrayIterator<ValueType, Extractor> IterType;
  135. typedef VarStreamArray<ValueType, Extractor> ArrayType;
  136. public:
  137. VarStreamArrayIterator(const ArrayType &Array, const Extractor &E,
  138. uint32_t Offset, bool *HadError)
  139. : IterRef(Array.Stream.drop_front(Offset)), Extract(E),
  140. Array(&Array), AbsOffset(Offset), HadError(HadError) {
  141. if (IterRef.getLength() == 0)
  142. moveToEnd();
  143. else {
  144. auto EC = Extract(IterRef, ThisLen, ThisValue);
  145. if (EC) {
  146. consumeError(std::move(EC));
  147. markError();
  148. }
  149. }
  150. }
  151. VarStreamArrayIterator() = default;
  152. explicit VarStreamArrayIterator(const Extractor &E) : Extract(E) {}
  153. ~VarStreamArrayIterator() = default;
  154. bool operator==(const IterType &R) const {
  155. if (Array && R.Array) {
  156. // Both have a valid array, make sure they're same.
  157. assert(Array == R.Array);
  158. return IterRef == R.IterRef;
  159. }
  160. // Both iterators are at the end.
  161. if (!Array && !R.Array)
  162. return true;
  163. // One is not at the end and one is.
  164. return false;
  165. }
  166. const ValueType &operator*() const {
  167. assert(Array && !HasError);
  168. return ThisValue;
  169. }
  170. ValueType &operator*() {
  171. assert(Array && !HasError);
  172. return ThisValue;
  173. }
  174. IterType &operator+=(unsigned N) {
  175. for (unsigned I = 0; I < N; ++I) {
  176. // We are done with the current record, discard it so that we are
  177. // positioned at the next record.
  178. AbsOffset += ThisLen;
  179. IterRef = IterRef.drop_front(ThisLen);
  180. if (IterRef.getLength() == 0) {
  181. // There is nothing after the current record, we must make this an end
  182. // iterator.
  183. moveToEnd();
  184. } else {
  185. // There is some data after the current record.
  186. auto EC = Extract(IterRef, ThisLen, ThisValue);
  187. if (EC) {
  188. consumeError(std::move(EC));
  189. markError();
  190. } else if (ThisLen == 0) {
  191. // An empty record? Make this an end iterator.
  192. moveToEnd();
  193. }
  194. }
  195. }
  196. return *this;
  197. }
  198. uint32_t offset() const { return AbsOffset; }
  199. uint32_t getRecordLength() const { return ThisLen; }
  200. private:
  201. void moveToEnd() {
  202. Array = nullptr;
  203. ThisLen = 0;
  204. }
  205. void markError() {
  206. moveToEnd();
  207. HasError = true;
  208. if (HadError != nullptr)
  209. *HadError = true;
  210. }
  211. ValueType ThisValue;
  212. BinaryStreamRef IterRef;
  213. Extractor Extract;
  214. const ArrayType *Array{nullptr};
  215. uint32_t ThisLen{0};
  216. uint32_t AbsOffset{0};
  217. bool HasError{false};
  218. bool *HadError{nullptr};
  219. };
  220. template <typename T> class FixedStreamArrayIterator;
  221. /// FixedStreamArray is similar to VarStreamArray, except with each record
  222. /// having a fixed-length. As with VarStreamArray, there is no upfront
  223. /// cost associated with building or copying a FixedStreamArray, as the
  224. /// memory for each element is not read from the backing stream until that
  225. /// element is iterated.
  226. template <typename T> class FixedStreamArray {
  227. friend class FixedStreamArrayIterator<T>;
  228. public:
  229. typedef FixedStreamArrayIterator<T> Iterator;
  230. FixedStreamArray() = default;
  231. explicit FixedStreamArray(BinaryStreamRef Stream) : Stream(Stream) {
  232. assert(Stream.getLength() % sizeof(T) == 0);
  233. }
  234. bool operator==(const FixedStreamArray<T> &Other) const {
  235. return Stream == Other.Stream;
  236. }
  237. bool operator!=(const FixedStreamArray<T> &Other) const {
  238. return !(*this == Other);
  239. }
  240. FixedStreamArray(const FixedStreamArray &) = default;
  241. FixedStreamArray &operator=(const FixedStreamArray &) = default;
  242. const T &operator[](uint32_t Index) const {
  243. assert(Index < size());
  244. uint32_t Off = Index * sizeof(T);
  245. ArrayRef<uint8_t> Data;
  246. if (auto EC = Stream.readBytes(Off, sizeof(T), Data)) {
  247. assert(false && "Unexpected failure reading from stream");
  248. // This should never happen since we asserted that the stream length was
  249. // an exact multiple of the element size.
  250. consumeError(std::move(EC));
  251. }
  252. assert(isAddrAligned(Align::Of<T>(), Data.data()));
  253. return *reinterpret_cast<const T *>(Data.data());
  254. }
  255. uint32_t size() const { return Stream.getLength() / sizeof(T); }
  256. bool empty() const { return size() == 0; }
  257. FixedStreamArrayIterator<T> begin() const {
  258. return FixedStreamArrayIterator<T>(*this, 0);
  259. }
  260. FixedStreamArrayIterator<T> end() const {
  261. return FixedStreamArrayIterator<T>(*this, size());
  262. }
  263. const T &front() const { return *begin(); }
  264. const T &back() const {
  265. FixedStreamArrayIterator<T> I = end();
  266. return *(--I);
  267. }
  268. BinaryStreamRef getUnderlyingStream() const { return Stream; }
  269. private:
  270. BinaryStreamRef Stream;
  271. };
  272. template <typename T>
  273. class FixedStreamArrayIterator
  274. : public iterator_facade_base<FixedStreamArrayIterator<T>,
  275. std::random_access_iterator_tag, const T> {
  276. public:
  277. FixedStreamArrayIterator(const FixedStreamArray<T> &Array, uint32_t Index)
  278. : Array(Array), Index(Index) {}
  279. FixedStreamArrayIterator<T>(const FixedStreamArrayIterator<T> &Other)
  280. : Array(Other.Array), Index(Other.Index) {}
  281. FixedStreamArrayIterator<T> &
  282. operator=(const FixedStreamArrayIterator<T> &Other) {
  283. Array = Other.Array;
  284. Index = Other.Index;
  285. return *this;
  286. }
  287. const T &operator*() const { return Array[Index]; }
  288. const T &operator*() { return Array[Index]; }
  289. bool operator==(const FixedStreamArrayIterator<T> &R) const {
  290. assert(Array == R.Array);
  291. return (Index == R.Index) && (Array == R.Array);
  292. }
  293. FixedStreamArrayIterator<T> &operator+=(std::ptrdiff_t N) {
  294. Index += N;
  295. return *this;
  296. }
  297. FixedStreamArrayIterator<T> &operator-=(std::ptrdiff_t N) {
  298. assert(std::ptrdiff_t(Index) >= N);
  299. Index -= N;
  300. return *this;
  301. }
  302. std::ptrdiff_t operator-(const FixedStreamArrayIterator<T> &R) const {
  303. assert(Array == R.Array);
  304. assert(Index >= R.Index);
  305. return Index - R.Index;
  306. }
  307. bool operator<(const FixedStreamArrayIterator<T> &RHS) const {
  308. assert(Array == RHS.Array);
  309. return Index < RHS.Index;
  310. }
  311. private:
  312. FixedStreamArray<T> Array;
  313. uint32_t Index;
  314. };
  315. } // namespace llvm
  316. #endif // LLVM_SUPPORT_BINARYSTREAMARRAY_H