VirtualFileSystem.h 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909
  1. //===- VirtualFileSystem.h - Virtual File System Layer ----------*- C++ -*-===//
  2. //
  3. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4. // See https://llvm.org/LICENSE.txt for license information.
  5. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6. //
  7. //===----------------------------------------------------------------------===//
  8. //
  9. /// \file
  10. /// Defines the virtual file system interface vfs::FileSystem.
  11. //
  12. //===----------------------------------------------------------------------===//
  13. #ifndef LLVM_SUPPORT_VIRTUALFILESYSTEM_H
  14. #define LLVM_SUPPORT_VIRTUALFILESYSTEM_H
  15. #include "llvm/ADT/IntrusiveRefCntPtr.h"
  16. #include "llvm/ADT/None.h"
  17. #include "llvm/ADT/Optional.h"
  18. #include "llvm/ADT/SmallVector.h"
  19. #include "llvm/ADT/StringRef.h"
  20. #include "llvm/Support/Chrono.h"
  21. #include "llvm/Support/ErrorOr.h"
  22. #include "llvm/Support/FileSystem.h"
  23. #include "llvm/Support/Path.h"
  24. #include "llvm/Support/SourceMgr.h"
  25. #include <cassert>
  26. #include <cstdint>
  27. #include <ctime>
  28. #include <memory>
  29. #include <stack>
  30. #include <string>
  31. #include <system_error>
  32. #include <utility>
  33. #include <vector>
  34. // ANDROID x86_64 defined the FS macro
  35. #undef FS
  36. namespace llvm {
  37. class MemoryBuffer;
  38. class MemoryBufferRef;
  39. class Twine;
  40. namespace vfs {
  41. /// The result of a \p status operation.
  42. class Status {
  43. std::string Name;
  44. llvm::sys::fs::UniqueID UID;
  45. llvm::sys::TimePoint<> MTime;
  46. uint32_t User;
  47. uint32_t Group;
  48. uint64_t Size;
  49. llvm::sys::fs::file_type Type = llvm::sys::fs::file_type::status_error;
  50. llvm::sys::fs::perms Perms;
  51. public:
  52. // FIXME: remove when files support multiple names
  53. bool IsVFSMapped = false;
  54. Status() = default;
  55. Status(const llvm::sys::fs::file_status &Status);
  56. Status(const Twine &Name, llvm::sys::fs::UniqueID UID,
  57. llvm::sys::TimePoint<> MTime, uint32_t User, uint32_t Group,
  58. uint64_t Size, llvm::sys::fs::file_type Type,
  59. llvm::sys::fs::perms Perms);
  60. /// Get a copy of a Status with a different name.
  61. static Status copyWithNewName(const Status &In, const Twine &NewName);
  62. static Status copyWithNewName(const llvm::sys::fs::file_status &In,
  63. const Twine &NewName);
  64. /// Returns the name that should be used for this file or directory.
  65. StringRef getName() const { return Name; }
  66. /// @name Status interface from llvm::sys::fs
  67. /// @{
  68. llvm::sys::fs::file_type getType() const { return Type; }
  69. llvm::sys::fs::perms getPermissions() const { return Perms; }
  70. llvm::sys::TimePoint<> getLastModificationTime() const { return MTime; }
  71. llvm::sys::fs::UniqueID getUniqueID() const { return UID; }
  72. uint32_t getUser() const { return User; }
  73. uint32_t getGroup() const { return Group; }
  74. uint64_t getSize() const { return Size; }
  75. /// @}
  76. /// @name Status queries
  77. /// These are static queries in llvm::sys::fs.
  78. /// @{
  79. bool equivalent(const Status &Other) const;
  80. bool isDirectory() const;
  81. bool isRegularFile() const;
  82. bool isOther() const;
  83. bool isSymlink() const;
  84. bool isStatusKnown() const;
  85. bool exists() const;
  86. /// @}
  87. };
  88. /// Represents an open file.
  89. class File {
  90. public:
  91. /// Destroy the file after closing it (if open).
  92. /// Sub-classes should generally call close() inside their destructors. We
  93. /// cannot do that from the base class, since close is virtual.
  94. virtual ~File();
  95. /// Get the status of the file.
  96. virtual llvm::ErrorOr<Status> status() = 0;
  97. /// Get the name of the file
  98. virtual llvm::ErrorOr<std::string> getName() {
  99. if (auto Status = status())
  100. return Status->getName().str();
  101. else
  102. return Status.getError();
  103. }
  104. /// Get the contents of the file as a \p MemoryBuffer.
  105. virtual llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
  106. getBuffer(const Twine &Name, int64_t FileSize = -1,
  107. bool RequiresNullTerminator = true, bool IsVolatile = false) = 0;
  108. /// Closes the file.
  109. virtual std::error_code close() = 0;
  110. };
  111. /// A member of a directory, yielded by a directory_iterator.
  112. /// Only information available on most platforms is included.
  113. class directory_entry {
  114. std::string Path;
  115. llvm::sys::fs::file_type Type = llvm::sys::fs::file_type::type_unknown;
  116. public:
  117. directory_entry() = default;
  118. directory_entry(std::string Path, llvm::sys::fs::file_type Type)
  119. : Path(std::move(Path)), Type(Type) {}
  120. llvm::StringRef path() const { return Path; }
  121. llvm::sys::fs::file_type type() const { return Type; }
  122. };
  123. namespace detail {
  124. /// An interface for virtual file systems to provide an iterator over the
  125. /// (non-recursive) contents of a directory.
  126. struct DirIterImpl {
  127. virtual ~DirIterImpl();
  128. /// Sets \c CurrentEntry to the next entry in the directory on success,
  129. /// to directory_entry() at end, or returns a system-defined \c error_code.
  130. virtual std::error_code increment() = 0;
  131. directory_entry CurrentEntry;
  132. };
  133. } // namespace detail
  134. /// An input iterator over the entries in a virtual path, similar to
  135. /// llvm::sys::fs::directory_iterator.
  136. class directory_iterator {
  137. std::shared_ptr<detail::DirIterImpl> Impl; // Input iterator semantics on copy
  138. public:
  139. directory_iterator(std::shared_ptr<detail::DirIterImpl> I)
  140. : Impl(std::move(I)) {
  141. assert(Impl.get() != nullptr && "requires non-null implementation");
  142. if (Impl->CurrentEntry.path().empty())
  143. Impl.reset(); // Normalize the end iterator to Impl == nullptr.
  144. }
  145. /// Construct an 'end' iterator.
  146. directory_iterator() = default;
  147. /// Equivalent to operator++, with an error code.
  148. directory_iterator &increment(std::error_code &EC) {
  149. assert(Impl && "attempting to increment past end");
  150. EC = Impl->increment();
  151. if (Impl->CurrentEntry.path().empty())
  152. Impl.reset(); // Normalize the end iterator to Impl == nullptr.
  153. return *this;
  154. }
  155. const directory_entry &operator*() const { return Impl->CurrentEntry; }
  156. const directory_entry *operator->() const { return &Impl->CurrentEntry; }
  157. bool operator==(const directory_iterator &RHS) const {
  158. if (Impl && RHS.Impl)
  159. return Impl->CurrentEntry.path() == RHS.Impl->CurrentEntry.path();
  160. return !Impl && !RHS.Impl;
  161. }
  162. bool operator!=(const directory_iterator &RHS) const {
  163. return !(*this == RHS);
  164. }
  165. };
  166. class FileSystem;
  167. namespace detail {
  168. /// Keeps state for the recursive_directory_iterator.
  169. struct RecDirIterState {
  170. std::stack<directory_iterator, std::vector<directory_iterator>> Stack;
  171. bool HasNoPushRequest = false;
  172. };
  173. } // end namespace detail
  174. /// An input iterator over the recursive contents of a virtual path,
  175. /// similar to llvm::sys::fs::recursive_directory_iterator.
  176. class recursive_directory_iterator {
  177. FileSystem *FS;
  178. std::shared_ptr<detail::RecDirIterState>
  179. State; // Input iterator semantics on copy.
  180. public:
  181. recursive_directory_iterator(FileSystem &FS, const Twine &Path,
  182. std::error_code &EC);
  183. /// Construct an 'end' iterator.
  184. recursive_directory_iterator() = default;
  185. /// Equivalent to operator++, with an error code.
  186. recursive_directory_iterator &increment(std::error_code &EC);
  187. const directory_entry &operator*() const { return *State->Stack.top(); }
  188. const directory_entry *operator->() const { return &*State->Stack.top(); }
  189. bool operator==(const recursive_directory_iterator &Other) const {
  190. return State == Other.State; // identity
  191. }
  192. bool operator!=(const recursive_directory_iterator &RHS) const {
  193. return !(*this == RHS);
  194. }
  195. /// Gets the current level. Starting path is at level 0.
  196. int level() const {
  197. assert(!State->Stack.empty() &&
  198. "Cannot get level without any iteration state");
  199. return State->Stack.size() - 1;
  200. }
  201. void no_push() { State->HasNoPushRequest = true; }
  202. };
  203. /// The virtual file system interface.
  204. class FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem> {
  205. public:
  206. virtual ~FileSystem();
  207. /// Get the status of the entry at \p Path, if one exists.
  208. virtual llvm::ErrorOr<Status> status(const Twine &Path) = 0;
  209. /// Get a \p File object for the file at \p Path, if one exists.
  210. virtual llvm::ErrorOr<std::unique_ptr<File>>
  211. openFileForRead(const Twine &Path) = 0;
  212. /// This is a convenience method that opens a file, gets its content and then
  213. /// closes the file.
  214. llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
  215. getBufferForFile(const Twine &Name, int64_t FileSize = -1,
  216. bool RequiresNullTerminator = true, bool IsVolatile = false);
  217. /// Get a directory_iterator for \p Dir.
  218. /// \note The 'end' iterator is directory_iterator().
  219. virtual directory_iterator dir_begin(const Twine &Dir,
  220. std::error_code &EC) = 0;
  221. /// Set the working directory. This will affect all following operations on
  222. /// this file system and may propagate down for nested file systems.
  223. virtual std::error_code setCurrentWorkingDirectory(const Twine &Path) = 0;
  224. /// Get the working directory of this file system.
  225. virtual llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const = 0;
  226. /// Gets real path of \p Path e.g. collapse all . and .. patterns, resolve
  227. /// symlinks. For real file system, this uses `llvm::sys::fs::real_path`.
  228. /// This returns errc::operation_not_permitted if not implemented by subclass.
  229. virtual std::error_code getRealPath(const Twine &Path,
  230. SmallVectorImpl<char> &Output) const;
  231. /// Check whether a file exists. Provided for convenience.
  232. bool exists(const Twine &Path);
  233. /// Is the file mounted on a local filesystem?
  234. virtual std::error_code isLocal(const Twine &Path, bool &Result);
  235. /// Make \a Path an absolute path.
  236. ///
  237. /// Makes \a Path absolute using the current directory if it is not already.
  238. /// An empty \a Path will result in the current directory.
  239. ///
  240. /// /absolute/path => /absolute/path
  241. /// relative/../path => <current-directory>/relative/../path
  242. ///
  243. /// \param Path A path that is modified to be an absolute path.
  244. /// \returns success if \a path has been made absolute, otherwise a
  245. /// platform-specific error_code.
  246. virtual std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const;
  247. };
  248. /// Gets an \p vfs::FileSystem for the 'real' file system, as seen by
  249. /// the operating system.
  250. /// The working directory is linked to the process's working directory.
  251. /// (This is usually thread-hostile).
  252. IntrusiveRefCntPtr<FileSystem> getRealFileSystem();
  253. /// Create an \p vfs::FileSystem for the 'real' file system, as seen by
  254. /// the operating system.
  255. /// It has its own working directory, independent of (but initially equal to)
  256. /// that of the process.
  257. std::unique_ptr<FileSystem> createPhysicalFileSystem();
  258. /// A file system that allows overlaying one \p AbstractFileSystem on top
  259. /// of another.
  260. ///
  261. /// Consists of a stack of >=1 \p FileSystem objects, which are treated as being
  262. /// one merged file system. When there is a directory that exists in more than
  263. /// one file system, the \p OverlayFileSystem contains a directory containing
  264. /// the union of their contents. The attributes (permissions, etc.) of the
  265. /// top-most (most recently added) directory are used. When there is a file
  266. /// that exists in more than one file system, the file in the top-most file
  267. /// system overrides the other(s).
  268. class OverlayFileSystem : public FileSystem {
  269. using FileSystemList = SmallVector<IntrusiveRefCntPtr<FileSystem>, 1>;
  270. /// The stack of file systems, implemented as a list in order of
  271. /// their addition.
  272. FileSystemList FSList;
  273. public:
  274. OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> Base);
  275. /// Pushes a file system on top of the stack.
  276. void pushOverlay(IntrusiveRefCntPtr<FileSystem> FS);
  277. llvm::ErrorOr<Status> status(const Twine &Path) override;
  278. llvm::ErrorOr<std::unique_ptr<File>>
  279. openFileForRead(const Twine &Path) override;
  280. directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
  281. llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
  282. std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
  283. std::error_code isLocal(const Twine &Path, bool &Result) override;
  284. std::error_code getRealPath(const Twine &Path,
  285. SmallVectorImpl<char> &Output) const override;
  286. using iterator = FileSystemList::reverse_iterator;
  287. using const_iterator = FileSystemList::const_reverse_iterator;
  288. using reverse_iterator = FileSystemList::iterator;
  289. using const_reverse_iterator = FileSystemList::const_iterator;
  290. /// Get an iterator pointing to the most recently added file system.
  291. iterator overlays_begin() { return FSList.rbegin(); }
  292. const_iterator overlays_begin() const { return FSList.rbegin(); }
  293. /// Get an iterator pointing one-past the least recently added file system.
  294. iterator overlays_end() { return FSList.rend(); }
  295. const_iterator overlays_end() const { return FSList.rend(); }
  296. /// Get an iterator pointing to the least recently added file system.
  297. reverse_iterator overlays_rbegin() { return FSList.begin(); }
  298. const_reverse_iterator overlays_rbegin() const { return FSList.begin(); }
  299. /// Get an iterator pointing one-past the most recently added file system.
  300. reverse_iterator overlays_rend() { return FSList.end(); }
  301. const_reverse_iterator overlays_rend() const { return FSList.end(); }
  302. };
  303. /// By default, this delegates all calls to the underlying file system. This
  304. /// is useful when derived file systems want to override some calls and still
  305. /// proxy other calls.
  306. class ProxyFileSystem : public FileSystem {
  307. public:
  308. explicit ProxyFileSystem(IntrusiveRefCntPtr<FileSystem> FS)
  309. : FS(std::move(FS)) {}
  310. llvm::ErrorOr<Status> status(const Twine &Path) override {
  311. return FS->status(Path);
  312. }
  313. llvm::ErrorOr<std::unique_ptr<File>>
  314. openFileForRead(const Twine &Path) override {
  315. return FS->openFileForRead(Path);
  316. }
  317. directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override {
  318. return FS->dir_begin(Dir, EC);
  319. }
  320. llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
  321. return FS->getCurrentWorkingDirectory();
  322. }
  323. std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
  324. return FS->setCurrentWorkingDirectory(Path);
  325. }
  326. std::error_code getRealPath(const Twine &Path,
  327. SmallVectorImpl<char> &Output) const override {
  328. return FS->getRealPath(Path, Output);
  329. }
  330. std::error_code isLocal(const Twine &Path, bool &Result) override {
  331. return FS->isLocal(Path, Result);
  332. }
  333. protected:
  334. FileSystem &getUnderlyingFS() { return *FS; }
  335. private:
  336. IntrusiveRefCntPtr<FileSystem> FS;
  337. virtual void anchor();
  338. };
  339. namespace detail {
  340. class InMemoryDirectory;
  341. class InMemoryFile;
  342. } // namespace detail
  343. /// An in-memory file system.
  344. class InMemoryFileSystem : public FileSystem {
  345. std::unique_ptr<detail::InMemoryDirectory> Root;
  346. std::string WorkingDirectory;
  347. bool UseNormalizedPaths = true;
  348. /// If HardLinkTarget is non-null, a hardlink is created to the To path which
  349. /// must be a file. If it is null then it adds the file as the public addFile.
  350. bool addFile(const Twine &Path, time_t ModificationTime,
  351. std::unique_ptr<llvm::MemoryBuffer> Buffer,
  352. Optional<uint32_t> User, Optional<uint32_t> Group,
  353. Optional<llvm::sys::fs::file_type> Type,
  354. Optional<llvm::sys::fs::perms> Perms,
  355. const detail::InMemoryFile *HardLinkTarget);
  356. public:
  357. explicit InMemoryFileSystem(bool UseNormalizedPaths = true);
  358. ~InMemoryFileSystem() override;
  359. /// Add a file containing a buffer or a directory to the VFS with a
  360. /// path. The VFS owns the buffer. If present, User, Group, Type
  361. /// and Perms apply to the newly-created file or directory.
  362. /// \return true if the file or directory was successfully added,
  363. /// false if the file or directory already exists in the file system with
  364. /// different contents.
  365. bool addFile(const Twine &Path, time_t ModificationTime,
  366. std::unique_ptr<llvm::MemoryBuffer> Buffer,
  367. Optional<uint32_t> User = None, Optional<uint32_t> Group = None,
  368. Optional<llvm::sys::fs::file_type> Type = None,
  369. Optional<llvm::sys::fs::perms> Perms = None);
  370. /// Add a hard link to a file.
  371. /// Here hard links are not intended to be fully equivalent to the classical
  372. /// filesystem. Both the hard link and the file share the same buffer and
  373. /// status (and thus have the same UniqueID). Because of this there is no way
  374. /// to distinguish between the link and the file after the link has been
  375. /// added.
  376. ///
  377. /// The To path must be an existing file or a hardlink. The From file must not
  378. /// have been added before. The To Path must not be a directory. The From Node
  379. /// is added as a hard link which points to the resolved file of To Node.
  380. /// \return true if the above condition is satisfied and hardlink was
  381. /// successfully created, false otherwise.
  382. bool addHardLink(const Twine &From, const Twine &To);
  383. /// Add a buffer to the VFS with a path. The VFS does not own the buffer.
  384. /// If present, User, Group, Type and Perms apply to the newly-created file
  385. /// or directory.
  386. /// \return true if the file or directory was successfully added,
  387. /// false if the file or directory already exists in the file system with
  388. /// different contents.
  389. bool addFileNoOwn(const Twine &Path, time_t ModificationTime,
  390. const llvm::MemoryBufferRef &Buffer,
  391. Optional<uint32_t> User = None,
  392. Optional<uint32_t> Group = None,
  393. Optional<llvm::sys::fs::file_type> Type = None,
  394. Optional<llvm::sys::fs::perms> Perms = None);
  395. std::string toString() const;
  396. /// Return true if this file system normalizes . and .. in paths.
  397. bool useNormalizedPaths() const { return UseNormalizedPaths; }
  398. llvm::ErrorOr<Status> status(const Twine &Path) override;
  399. llvm::ErrorOr<std::unique_ptr<File>>
  400. openFileForRead(const Twine &Path) override;
  401. directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
  402. llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
  403. return WorkingDirectory;
  404. }
  405. /// Canonicalizes \p Path by combining with the current working
  406. /// directory and normalizing the path (e.g. remove dots). If the current
  407. /// working directory is not set, this returns errc::operation_not_permitted.
  408. ///
  409. /// This doesn't resolve symlinks as they are not supported in in-memory file
  410. /// system.
  411. std::error_code getRealPath(const Twine &Path,
  412. SmallVectorImpl<char> &Output) const override;
  413. std::error_code isLocal(const Twine &Path, bool &Result) override;
  414. std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
  415. };
  416. /// Get a globally unique ID for a virtual file or directory.
  417. llvm::sys::fs::UniqueID getNextVirtualUniqueID();
  418. /// Gets a \p FileSystem for a virtual file system described in YAML
  419. /// format.
  420. std::unique_ptr<FileSystem>
  421. getVFSFromYAML(std::unique_ptr<llvm::MemoryBuffer> Buffer,
  422. llvm::SourceMgr::DiagHandlerTy DiagHandler,
  423. StringRef YAMLFilePath, void *DiagContext = nullptr,
  424. IntrusiveRefCntPtr<FileSystem> ExternalFS = getRealFileSystem());
  425. struct YAMLVFSEntry {
  426. template <typename T1, typename T2>
  427. YAMLVFSEntry(T1 &&VPath, T2 &&RPath, bool IsDirectory = false)
  428. : VPath(std::forward<T1>(VPath)), RPath(std::forward<T2>(RPath)),
  429. IsDirectory(IsDirectory) {}
  430. std::string VPath;
  431. std::string RPath;
  432. bool IsDirectory = false;
  433. };
  434. class RedirectingFSDirIterImpl;
  435. class RedirectingFileSystemParser;
  436. /// A virtual file system parsed from a YAML file.
  437. ///
  438. /// Currently, this class allows creating virtual files and directories. Virtual
  439. /// files map to existing external files in \c ExternalFS, and virtual
  440. /// directories may either map to existing directories in \c ExternalFS or list
  441. /// their contents in the form of other virtual directories and/or files.
  442. ///
  443. /// The basic structure of the parsed file is:
  444. /// \verbatim
  445. /// {
  446. /// 'version': <version number>,
  447. /// <optional configuration>
  448. /// 'roots': [
  449. /// <directory entries>
  450. /// ]
  451. /// }
  452. /// \endverbatim
  453. ///
  454. /// All configuration options are optional.
  455. /// 'case-sensitive': <boolean, default=(true for Posix, false for Windows)>
  456. /// 'use-external-names': <boolean, default=true>
  457. /// 'overlay-relative': <boolean, default=false>
  458. /// 'fallthrough': <boolean, default=true>
  459. ///
  460. /// Virtual directories that list their contents are represented as
  461. /// \verbatim
  462. /// {
  463. /// 'type': 'directory',
  464. /// 'name': <string>,
  465. /// 'contents': [ <file or directory entries> ]
  466. /// }
  467. /// \endverbatim
  468. ///
  469. /// The default attributes for such virtual directories are:
  470. /// \verbatim
  471. /// MTime = now() when created
  472. /// Perms = 0777
  473. /// User = Group = 0
  474. /// Size = 0
  475. /// UniqueID = unspecified unique value
  476. /// \endverbatim
  477. ///
  478. /// When a path prefix matches such a directory, the next component in the path
  479. /// is matched against the entries in the 'contents' array.
  480. ///
  481. /// Re-mapped directories, on the other hand, are represented as
  482. /// /// \verbatim
  483. /// {
  484. /// 'type': 'directory-remap',
  485. /// 'name': <string>,
  486. /// 'use-external-name': <boolean>, # Optional
  487. /// 'external-contents': <path to external directory>
  488. /// }
  489. /// \endverbatim
  490. ///
  491. /// and inherit their attributes from the external directory. When a path
  492. /// prefix matches such an entry, the unmatched components are appended to the
  493. /// 'external-contents' path, and the resulting path is looked up in the
  494. /// external file system instead.
  495. ///
  496. /// Re-mapped files are represented as
  497. /// \verbatim
  498. /// {
  499. /// 'type': 'file',
  500. /// 'name': <string>,
  501. /// 'use-external-name': <boolean>, # Optional
  502. /// 'external-contents': <path to external file>
  503. /// }
  504. /// \endverbatim
  505. ///
  506. /// Their attributes and file contents are determined by looking up the file at
  507. /// their 'external-contents' path in the external file system.
  508. ///
  509. /// For 'file', 'directory' and 'directory-remap' entries the 'name' field may
  510. /// contain multiple path components (e.g. /path/to/file). However, any
  511. /// directory in such a path that contains more than one child must be uniquely
  512. /// represented by a 'directory' entry.
  513. class RedirectingFileSystem : public vfs::FileSystem {
  514. public:
  515. enum EntryKind { EK_Directory, EK_DirectoryRemap, EK_File };
  516. enum NameKind { NK_NotSet, NK_External, NK_Virtual };
  517. /// A single file or directory in the VFS.
  518. class Entry {
  519. EntryKind Kind;
  520. std::string Name;
  521. public:
  522. Entry(EntryKind K, StringRef Name) : Kind(K), Name(Name) {}
  523. virtual ~Entry() = default;
  524. StringRef getName() const { return Name; }
  525. EntryKind getKind() const { return Kind; }
  526. };
  527. /// A directory in the vfs with explicitly specified contents.
  528. class DirectoryEntry : public Entry {
  529. std::vector<std::unique_ptr<Entry>> Contents;
  530. Status S;
  531. public:
  532. /// Constructs a directory entry with explicitly specified contents.
  533. DirectoryEntry(StringRef Name, std::vector<std::unique_ptr<Entry>> Contents,
  534. Status S)
  535. : Entry(EK_Directory, Name), Contents(std::move(Contents)),
  536. S(std::move(S)) {}
  537. /// Constructs an empty directory entry.
  538. DirectoryEntry(StringRef Name, Status S)
  539. : Entry(EK_Directory, Name), S(std::move(S)) {}
  540. Status getStatus() { return S; }
  541. void addContent(std::unique_ptr<Entry> Content) {
  542. Contents.push_back(std::move(Content));
  543. }
  544. Entry *getLastContent() const { return Contents.back().get(); }
  545. using iterator = decltype(Contents)::iterator;
  546. iterator contents_begin() { return Contents.begin(); }
  547. iterator contents_end() { return Contents.end(); }
  548. static bool classof(const Entry *E) { return E->getKind() == EK_Directory; }
  549. };
  550. /// A file or directory in the vfs that is mapped to a file or directory in
  551. /// the external filesystem.
  552. class RemapEntry : public Entry {
  553. std::string ExternalContentsPath;
  554. NameKind UseName;
  555. protected:
  556. RemapEntry(EntryKind K, StringRef Name, StringRef ExternalContentsPath,
  557. NameKind UseName)
  558. : Entry(K, Name), ExternalContentsPath(ExternalContentsPath),
  559. UseName(UseName) {}
  560. public:
  561. StringRef getExternalContentsPath() const { return ExternalContentsPath; }
  562. /// Whether to use the external path as the name for this file or directory.
  563. bool useExternalName(bool GlobalUseExternalName) const {
  564. return UseName == NK_NotSet ? GlobalUseExternalName
  565. : (UseName == NK_External);
  566. }
  567. NameKind getUseName() const { return UseName; }
  568. static bool classof(const Entry *E) {
  569. switch (E->getKind()) {
  570. case EK_DirectoryRemap:
  571. LLVM_FALLTHROUGH;
  572. case EK_File:
  573. return true;
  574. case EK_Directory:
  575. return false;
  576. }
  577. llvm_unreachable("invalid entry kind");
  578. }
  579. };
  580. /// A directory in the vfs that maps to a directory in the external file
  581. /// system.
  582. class DirectoryRemapEntry : public RemapEntry {
  583. public:
  584. DirectoryRemapEntry(StringRef Name, StringRef ExternalContentsPath,
  585. NameKind UseName)
  586. : RemapEntry(EK_DirectoryRemap, Name, ExternalContentsPath, UseName) {}
  587. static bool classof(const Entry *E) {
  588. return E->getKind() == EK_DirectoryRemap;
  589. }
  590. };
  591. /// A file in the vfs that maps to a file in the external file system.
  592. class FileEntry : public RemapEntry {
  593. public:
  594. FileEntry(StringRef Name, StringRef ExternalContentsPath, NameKind UseName)
  595. : RemapEntry(EK_File, Name, ExternalContentsPath, UseName) {}
  596. static bool classof(const Entry *E) { return E->getKind() == EK_File; }
  597. };
  598. /// Represents the result of a path lookup into the RedirectingFileSystem.
  599. struct LookupResult {
  600. /// The entry the looked-up path corresponds to.
  601. Entry *E;
  602. private:
  603. /// When the found Entry is a DirectoryRemapEntry, stores the path in the
  604. /// external file system that the looked-up path in the virtual file system
  605. // corresponds to.
  606. Optional<std::string> ExternalRedirect;
  607. public:
  608. LookupResult(Entry *E, sys::path::const_iterator Start,
  609. sys::path::const_iterator End);
  610. /// If the found Entry maps the the input path to a path in the external
  611. /// file system (i.e. it is a FileEntry or DirectoryRemapEntry), returns
  612. /// that path.
  613. Optional<StringRef> getExternalRedirect() const {
  614. if (isa<DirectoryRemapEntry>(E))
  615. return StringRef(*ExternalRedirect);
  616. if (auto *FE = dyn_cast<FileEntry>(E))
  617. return FE->getExternalContentsPath();
  618. return None;
  619. }
  620. };
  621. private:
  622. friend class RedirectingFSDirIterImpl;
  623. friend class RedirectingFileSystemParser;
  624. bool shouldUseExternalFS() const { return IsFallthrough; }
  625. /// Canonicalize path by removing ".", "..", "./", components. This is
  626. /// a VFS request, do not bother about symlinks in the path components
  627. /// but canonicalize in order to perform the correct entry search.
  628. std::error_code makeCanonical(SmallVectorImpl<char> &Path) const;
  629. /// Whether to fall back to the external file system when an operation fails
  630. /// with the given error code on a path associated with the provided Entry.
  631. bool shouldFallBackToExternalFS(std::error_code EC, Entry *E = nullptr) const;
  632. // In a RedirectingFileSystem, keys can be specified in Posix or Windows
  633. // style (or even a mixture of both), so this comparison helper allows
  634. // slashes (representing a root) to match backslashes (and vice versa). Note
  635. // that, other than the root, path components should not contain slashes or
  636. // backslashes.
  637. bool pathComponentMatches(llvm::StringRef lhs, llvm::StringRef rhs) const {
  638. if ((CaseSensitive ? lhs.equals(rhs) : lhs.equals_lower(rhs)))
  639. return true;
  640. return (lhs == "/" && rhs == "\\") || (lhs == "\\" && rhs == "/");
  641. }
  642. /// The root(s) of the virtual file system.
  643. std::vector<std::unique_ptr<Entry>> Roots;
  644. /// The current working directory of the file system.
  645. std::string WorkingDirectory;
  646. /// The file system to use for external references.
  647. IntrusiveRefCntPtr<FileSystem> ExternalFS;
  648. /// If IsRelativeOverlay is set, this represents the directory
  649. /// path that should be prefixed to each 'external-contents' entry
  650. /// when reading from YAML files.
  651. std::string ExternalContentsPrefixDir;
  652. /// @name Configuration
  653. /// @{
  654. /// Whether to perform case-sensitive comparisons.
  655. ///
  656. /// Currently, case-insensitive matching only works correctly with ASCII.
  657. bool CaseSensitive =
  658. #ifdef _WIN32
  659. false;
  660. #else
  661. true;
  662. #endif
  663. /// IsRelativeOverlay marks whether a ExternalContentsPrefixDir path must
  664. /// be prefixed in every 'external-contents' when reading from YAML files.
  665. bool IsRelativeOverlay = false;
  666. /// Whether to use to use the value of 'external-contents' for the
  667. /// names of files. This global value is overridable on a per-file basis.
  668. bool UseExternalNames = true;
  669. /// Whether to attempt a file lookup in external file system after it wasn't
  670. /// found in VFS.
  671. bool IsFallthrough = true;
  672. /// @}
  673. RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS);
  674. /// Looks up the path <tt>[Start, End)</tt> in \p From, possibly recursing
  675. /// into the contents of \p From if it is a directory. Returns a LookupResult
  676. /// giving the matched entry and, if that entry is a FileEntry or
  677. /// DirectoryRemapEntry, the path it redirects to in the external file system.
  678. ErrorOr<LookupResult> lookupPathImpl(llvm::sys::path::const_iterator Start,
  679. llvm::sys::path::const_iterator End,
  680. Entry *From) const;
  681. /// Get the status for a path with the provided \c LookupResult.
  682. ErrorOr<Status> status(const Twine &Path, const LookupResult &Result);
  683. public:
  684. /// Looks up \p Path in \c Roots and returns a LookupResult giving the
  685. /// matched entry and, if the entry was a FileEntry or DirectoryRemapEntry,
  686. /// the path it redirects to in the external file system.
  687. ErrorOr<LookupResult> lookupPath(StringRef Path) const;
  688. /// Parses \p Buffer, which is expected to be in YAML format and
  689. /// returns a virtual file system representing its contents.
  690. static std::unique_ptr<RedirectingFileSystem>
  691. create(std::unique_ptr<MemoryBuffer> Buffer,
  692. SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
  693. void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
  694. /// Redirect each of the remapped files from first to second.
  695. static std::unique_ptr<RedirectingFileSystem>
  696. create(ArrayRef<std::pair<std::string, std::string>> RemappedFiles,
  697. bool UseExternalNames, FileSystem &ExternalFS);
  698. ErrorOr<Status> status(const Twine &Path) override;
  699. ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
  700. std::error_code getRealPath(const Twine &Path,
  701. SmallVectorImpl<char> &Output) const override;
  702. llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
  703. std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
  704. std::error_code isLocal(const Twine &Path, bool &Result) override;
  705. std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const override;
  706. directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
  707. void setExternalContentsPrefixDir(StringRef PrefixDir);
  708. StringRef getExternalContentsPrefixDir() const;
  709. void setFallthrough(bool Fallthrough);
  710. std::vector<llvm::StringRef> getRoots() const;
  711. void dump(raw_ostream &OS) const;
  712. void dumpEntry(raw_ostream &OS, Entry *E, int NumSpaces = 0) const;
  713. #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
  714. LLVM_DUMP_METHOD void dump() const;
  715. #endif
  716. };
  717. /// Collect all pairs of <virtual path, real path> entries from the
  718. /// \p YAMLFilePath. This is used by the module dependency collector to forward
  719. /// the entries into the reproducer output VFS YAML file.
  720. void collectVFSFromYAML(
  721. std::unique_ptr<llvm::MemoryBuffer> Buffer,
  722. llvm::SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
  723. SmallVectorImpl<YAMLVFSEntry> &CollectedEntries,
  724. void *DiagContext = nullptr,
  725. IntrusiveRefCntPtr<FileSystem> ExternalFS = getRealFileSystem());
  726. class YAMLVFSWriter {
  727. std::vector<YAMLVFSEntry> Mappings;
  728. Optional<bool> IsCaseSensitive;
  729. Optional<bool> IsOverlayRelative;
  730. Optional<bool> UseExternalNames;
  731. std::string OverlayDir;
  732. void addEntry(StringRef VirtualPath, StringRef RealPath, bool IsDirectory);
  733. public:
  734. YAMLVFSWriter() = default;
  735. void addFileMapping(StringRef VirtualPath, StringRef RealPath);
  736. void addDirectoryMapping(StringRef VirtualPath, StringRef RealPath);
  737. void setCaseSensitivity(bool CaseSensitive) {
  738. IsCaseSensitive = CaseSensitive;
  739. }
  740. void setUseExternalNames(bool UseExtNames) { UseExternalNames = UseExtNames; }
  741. void setOverlayDir(StringRef OverlayDirectory) {
  742. IsOverlayRelative = true;
  743. OverlayDir.assign(OverlayDirectory.str());
  744. }
  745. const std::vector<YAMLVFSEntry> &getMappings() const { return Mappings; }
  746. void write(llvm::raw_ostream &OS);
  747. };
  748. } // namespace vfs
  749. } // namespace llvm
  750. #endif // LLVM_SUPPORT_VIRTUALFILESYSTEM_H