1//===-- FileCollector.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Support/FileCollector.h"
10#include "llvm/ADT/SmallString.h"
11#include "llvm/ADT/Twine.h"
12#include "llvm/Support/FileSystem.h"
13#include "llvm/Support/Path.h"
14#include "llvm/Support/Process.h"
15
16using namespace llvm;
17
18FileCollectorBase::FileCollectorBase() = default;
19FileCollectorBase::~FileCollectorBase() = default;
20
21void FileCollectorBase::addFile(const Twine &File) {
22 std::lock_guard<std::mutex> lock(Mutex);
23 std::string FileStr = File.str();
24 if (markAsSeen(Path: FileStr))
25 addFileImpl(SrcPath: FileStr);
26}
27
28void FileCollectorBase::addDirectory(const Twine &Dir) {
29 assert(sys::fs::is_directory(Dir));
30 std::error_code EC;
31 addDirectoryImpl(Dir, FS: vfs::getRealFileSystem(), EC);
32}
33
34static bool isCaseSensitivePath(StringRef Path) {
35 SmallString<256> TmpDest = Path, UpperDest, RealDest;
36
37 // Remove component traversals, links, etc.
38 if (sys::fs::real_path(path: Path, output&: TmpDest))
39 return true; // Current default value in vfs.yaml
40 Path = TmpDest;
41
42 // Change path to all upper case and ask for its real path, if the latter
43 // exists and is equal to path, it's not case sensitive. Default to case
44 // sensitive in the absence of real_path, since this is the YAMLVFSWriter
45 // default.
46 UpperDest = Path.upper();
47 if (!sys::fs::real_path(path: UpperDest, output&: RealDest) && Path == RealDest)
48 return false;
49 return true;
50}
51
52FileCollector::FileCollector(std::string Root, std::string OverlayRoot)
53 : Root(Root), OverlayRoot(OverlayRoot) {
54 assert(sys::path::is_absolute(Root) && "Root not absolute");
55 assert(sys::path::is_absolute(OverlayRoot) && "OverlayRoot not absolute");
56}
57
58void FileCollector::PathCanonicalizer::updateWithRealPath(
59 SmallVectorImpl<char> &Path) {
60 StringRef SrcPath(Path.begin(), Path.size());
61 StringRef Filename = sys::path::filename(path: SrcPath);
62 StringRef Directory = sys::path::parent_path(path: SrcPath);
63
64 // Use real_path to fix any symbolic link component present in the directory
65 // part of the path, caching the search because computing the real path is
66 // expensive.
67 SmallString<256> RealPath;
68 auto DirWithSymlink = CachedDirs.find(Key: Directory);
69 if (DirWithSymlink == CachedDirs.end()) {
70 // FIXME: Should this be a call to FileSystem::getRealpath(), in some
71 // cases? What if there is nothing on disk?
72 if (sys::fs::real_path(path: Directory, output&: RealPath))
73 return;
74 CachedDirs[Directory] = std::string(RealPath);
75 } else {
76 RealPath = DirWithSymlink->second;
77 }
78
79 // Finish recreating the path by appending the original filename, since we
80 // don't need to resolve symlinks in the filename.
81 //
82 // FIXME: If we can cope with this, maybe we can cope without calling
83 // getRealPath() at all when there's no ".." component.
84 sys::path::append(path&: RealPath, a: Filename);
85
86 // Swap to create the output.
87 Path.swap(RHS&: RealPath);
88}
89
90/// Make Path absolute.
91static void makeAbsolute(SmallVectorImpl<char> &Path) {
92 // We need an absolute src path to append to the root.
93 sys::fs::make_absolute(path&: Path);
94
95 // Canonicalize src to a native path to avoid mixed separator styles.
96 sys::path::native(path&: Path);
97
98 // Remove redundant leading "./" pieces and consecutive separators.
99 Path.erase(CS: Path.begin(), CE: sys::path::remove_leading_dotslash(
100 path: StringRef(Path.begin(), Path.size()))
101 .begin());
102}
103
104FileCollector::PathCanonicalizer::PathStorage
105FileCollector::PathCanonicalizer::canonicalize(StringRef SrcPath) {
106 PathStorage Paths;
107 Paths.VirtualPath = SrcPath;
108 makeAbsolute(Path&: Paths.VirtualPath);
109
110 // If a ".." component is present after a symlink component, remove_dots may
111 // lead to the wrong real destination path. Let the source be canonicalized
112 // like that but make sure we always use the real path for the destination.
113 Paths.CopyFrom = Paths.VirtualPath;
114 updateWithRealPath(Path&: Paths.CopyFrom);
115
116 // Canonicalize the virtual path by removing "..", "." components.
117 sys::path::remove_dots(path&: Paths.VirtualPath, /*remove_dot_dot=*/true);
118
119 return Paths;
120}
121
122void FileCollector::addFileImpl(StringRef SrcPath) {
123 PathCanonicalizer::PathStorage Paths = Canonicalizer.canonicalize(SrcPath);
124
125 SmallString<256> DstPath = StringRef(Root);
126 sys::path::append(path&: DstPath, a: sys::path::relative_path(path: Paths.CopyFrom));
127
128 // Always map a canonical src path to its real path into the YAML, by doing
129 // this we map different virtual src paths to the same entry in the VFS
130 // overlay, which is a way to emulate symlink inside the VFS; this is also
131 // needed for correctness, not doing that can lead to module redefinition
132 // errors.
133 addFileToMapping(VirtualPath: Paths.VirtualPath, RealPath: DstPath);
134}
135
136llvm::vfs::directory_iterator
137FileCollector::addDirectoryImpl(const llvm::Twine &Dir,
138 IntrusiveRefCntPtr<vfs::FileSystem> FS,
139 std::error_code &EC) {
140 auto It = FS->dir_begin(Dir, EC);
141 if (EC)
142 return It;
143 addFile(File: Dir);
144 for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
145 if (It->type() == sys::fs::file_type::regular_file ||
146 It->type() == sys::fs::file_type::directory_file ||
147 It->type() == sys::fs::file_type::symlink_file) {
148 addFile(File: It->path());
149 }
150 }
151 if (EC)
152 return It;
153 // Return a new iterator.
154 return FS->dir_begin(Dir, EC);
155}
156
157/// Set the access and modification time for the given file from the given
158/// status object.
159static std::error_code
160copyAccessAndModificationTime(StringRef Filename,
161 const sys::fs::file_status &Stat) {
162 int FD;
163
164 if (auto EC =
165 sys::fs::openFileForWrite(Name: Filename, ResultFD&: FD, Disp: sys::fs::CD_OpenExisting))
166 return EC;
167
168 if (auto EC = sys::fs::setLastAccessAndModificationTime(
169 FD, AccessTime: Stat.getLastAccessedTime(), ModificationTime: Stat.getLastModificationTime()))
170 return EC;
171
172 if (auto EC = sys::Process::SafelyCloseFileDescriptor(FD))
173 return EC;
174
175 return {};
176}
177
178std::error_code FileCollector::copyFiles(bool StopOnError) {
179 auto Err = sys::fs::create_directories(path: Root, /*IgnoreExisting=*/true);
180 if (Err) {
181 return Err;
182 }
183
184 std::lock_guard<std::mutex> lock(Mutex);
185
186 for (auto &entry : VFSWriter.getMappings()) {
187 // Get the status of the original file/directory.
188 sys::fs::file_status Stat;
189 if (std::error_code EC = sys::fs::status(path: entry.VPath, result&: Stat)) {
190 if (StopOnError)
191 return EC;
192 continue;
193 }
194
195 // Continue if the file doesn't exist.
196 if (Stat.type() == sys::fs::file_type::file_not_found)
197 continue;
198
199 // Create directory tree.
200 if (std::error_code EC =
201 sys::fs::create_directories(path: sys::path::parent_path(path: entry.RPath),
202 /*IgnoreExisting=*/true)) {
203 if (StopOnError)
204 return EC;
205 }
206
207 if (Stat.type() == sys::fs::file_type::directory_file) {
208 // Construct a directory when it's just a directory entry.
209 if (std::error_code EC =
210 sys::fs::create_directories(path: entry.RPath,
211 /*IgnoreExisting=*/true)) {
212 if (StopOnError)
213 return EC;
214 }
215 continue;
216 }
217
218 // Copy file over.
219 if (std::error_code EC = sys::fs::copy_file(From: entry.VPath, To: entry.RPath)) {
220 if (StopOnError)
221 return EC;
222 }
223
224 // Copy over permissions.
225 if (auto perms = sys::fs::getPermissions(Path: entry.VPath)) {
226 if (std::error_code EC = sys::fs::setPermissions(Path: entry.RPath, Permissions: *perms)) {
227 if (StopOnError)
228 return EC;
229 }
230 }
231
232 // Copy over modification time.
233 copyAccessAndModificationTime(Filename: entry.RPath, Stat);
234 }
235 return {};
236}
237
238std::error_code FileCollector::writeMapping(StringRef MappingFile) {
239 std::lock_guard<std::mutex> lock(Mutex);
240
241 VFSWriter.setOverlayDir(OverlayRoot);
242 VFSWriter.setCaseSensitivity(isCaseSensitivePath(Path: OverlayRoot));
243 VFSWriter.setUseExternalNames(false);
244
245 std::error_code EC;
246 raw_fd_ostream os(MappingFile, EC, sys::fs::OF_TextWithCRLF);
247 if (EC)
248 return EC;
249
250 VFSWriter.write(OS&: os);
251
252 return {};
253}
254
255namespace llvm {
256
257class FileCollectorFileSystem : public vfs::FileSystem {
258public:
259 explicit FileCollectorFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS,
260 std::shared_ptr<FileCollector> Collector)
261 : FS(std::move(FS)), Collector(std::move(Collector)) {}
262
263 llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override {
264 auto Result = FS->status(Path);
265 if (Result && Result->exists())
266 Collector->addFile(File: Path);
267 return Result;
268 }
269
270 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
271 openFileForRead(const Twine &Path) override {
272 auto Result = FS->openFileForRead(Path);
273 if (Result && *Result)
274 Collector->addFile(File: Path);
275 return Result;
276 }
277
278 llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
279 std::error_code &EC) override {
280 return Collector->addDirectoryImpl(Dir, FS, EC);
281 }
282
283 std::error_code getRealPath(const Twine &Path,
284 SmallVectorImpl<char> &Output) override {
285 auto EC = FS->getRealPath(Path, Output);
286 if (!EC) {
287 Collector->addFile(File: Path);
288 if (Output.size() > 0)
289 Collector->addFile(File: Output);
290 }
291 return EC;
292 }
293
294 std::error_code isLocal(const Twine &Path, bool &Result) override {
295 return FS->isLocal(Path, Result);
296 }
297
298 llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
299 return FS->getCurrentWorkingDirectory();
300 }
301
302 std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
303 return FS->setCurrentWorkingDirectory(Path);
304 }
305
306private:
307 IntrusiveRefCntPtr<vfs::FileSystem> FS;
308 std::shared_ptr<FileCollector> Collector;
309};
310
311} // namespace llvm
312
313IntrusiveRefCntPtr<vfs::FileSystem>
314FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,
315 std::shared_ptr<FileCollector> Collector) {
316 return new FileCollectorFileSystem(std::move(BaseFS), std::move(Collector));
317}
318