Skip to content

Commit

Permalink
Add new -C option to specify a separately-compiled unit file listing
Browse files Browse the repository at this point in the history
  • Loading branch information
@MikePopoloski
MikePopoloski committed Feb 4, 2024
1 parent e700601 commit 30d2d67
Show file tree
Hide file tree
Showing 8 changed files with 235 additions and 23 deletions.
5 changes: 4 additions & 1 deletion bindings/python/CompBindings.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,8 @@ void registerCompilation(py::module_& m) {
return self.parseCommandLine(arg, parseOptions);
},
"arg"_a, "parseOptions"_a = CommandLine::ParseOptions{})
.def("processCommandFiles", &Driver::processCommandFiles, "fileName"_a, "makeRelative"_a)
.def("processCommandFiles", &Driver::processCommandFiles, "fileName"_a, "makeRelative"_a,
"separateUnit"_a)
.def("processOptions", &Driver::processOptions)
.def("runPreprocessor", &Driver::runPreprocessor, "includeComments"_a,
"includeDirectives"_a, "obfuscateIds"_a, "useFixedObfuscationSeed"_a = false)
Expand All @@ -196,6 +197,8 @@ void registerCompilation(py::module_& m) {
.def("addSearchExtension", &SourceLoader::addSearchExtension, "extension"_a)
.def("addLibraryMaps", &SourceLoader::addLibraryMaps, "pattern"_a, "basePath"_a,
"optionBag"_a)
.def("addSeparateUnit", &SourceLoader::addSeparateUnit, "filePatterns"_a, "includePaths"_a,
"defines"_a, "libraryName"_a)
.def("loadSources", &SourceLoader::loadSources)
.def("loadAndParseSources", &SourceLoader::loadAndParseSources, "optionBag"_a)
.def_property_readonly("hasFiles", &SourceLoader::hasFiles)
Expand Down
6 changes: 5 additions & 1 deletion include/slang/driver/Driver.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,8 +250,11 @@ class SLANG_EXPORT Driver {
/// @param pattern a file path pattern indicating the command file(s) to process.
/// @param makeRelative indicates whether paths in the file are relative to the file
/// itself or to the current working directory.
/// @param separateUnit if true, the file is a separate compilation unit listing;
/// options within it apply only to that unit and not the
/// broader compilation.
/// @returns true on success and false if errors were encountered.
bool processCommandFiles(std::string_view pattern, bool makeRelative);
bool processCommandFiles(std::string_view pattern, bool makeRelative, bool separateUnit);

/// Processes and applies all configured options.
/// @returns true on success and false if errors were encountered.
Expand Down Expand Up @@ -297,6 +300,7 @@ class SLANG_EXPORT Driver {
[[nodiscard]] bool reportCompilation(ast::Compilation& compilation, bool quiet);

private:
bool parseUnitListing(std::string_view text);
void addLibraryFiles(std::string_view pattern);
void addParseOptions(Bag& bag) const;
void addCompilationOptions(Bag& bag) const;
Expand Down
37 changes: 33 additions & 4 deletions include/slang/driver/SourceLoader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
//------------------------------------------------------------------------------
#pragma once

#include <deque>
#include <filesystem>
#include <memory>
#include <mutex>
Expand Down Expand Up @@ -110,6 +111,19 @@ class SLANG_EXPORT SourceLoader {
void addLibraryMaps(std::string_view pattern, const std::filesystem::path& basePath,
const Bag& optionBag);

/// @brief Adds a group of files as a separately compiled compilation unit.
///
/// Unlike files added via the @a addFiles method, files added here are
/// all guaranteed to be grouped into a single compilation unit and use
/// the provided options for preprocessor defines and include paths.
///
/// If the library name is provided the compilation unit will be included
/// in the library of that name; otherwise it will be included in the
/// default library and be considered a non-library unit.
void addSeparateUnit(std::span<const std::string> filePatterns,
std::vector<std::string> includePaths, std::vector<std::string> defines,
std::string libraryName);

/// Returns a list of all library map syntax trees that have been loaded and parsed.
const SyntaxTreeList& getLibraryMaps() const { return libraryMapTrees; }

Expand All @@ -128,6 +142,14 @@ class SLANG_EXPORT SourceLoader {
std::span<const std::string> getErrors() const { return errors; }

private:
// One entry per unit of files + options to compile them.
// Only used for addSeparateUnit.
struct UnitEntry {
std::vector<std::string> includePaths;
std::vector<std::string> defines;
const SourceLibrary* library = nullptr;
};

// One entry per unique file path added to the loader.
struct FileEntry {
// The filesystem path (as specified by the user).
Expand All @@ -144,6 +166,9 @@ class SLANG_EXPORT SourceLoader {
// matches at an even higher rank.
const SourceLibrary* secondLib = nullptr;

// A pointer to the unit this file is a part of, or nullptr if none.
const UnitEntry* unit = nullptr;

// A measure of how strongly this file belongs to the library.
GlobRank libraryRank;

Expand All @@ -156,22 +181,25 @@ class SLANG_EXPORT SourceLoader {
bool isLibraryFile = false;

FileEntry(std::filesystem::path&& path, bool isLibraryFile, const SourceLibrary* library,
GlobRank libraryRank) :
const UnitEntry* unit, GlobRank libraryRank) :
path(std::move(path)),
library(library), libraryRank(libraryRank), isLibraryFile(isLibraryFile) {}
library(library), unit(unit), libraryRank(libraryRank), isLibraryFile(isLibraryFile) {}
};

// The result of a loadAndParse call.
// 0: A parsed syntax tree
// 1: A loaded source buffer + bool that indicates whether it's a library
// 2: A file entry + error code if the load fails
// 3: A source buffer + unit pointer if it's part of a separate unit
using LoadResult =
std::variant<std::shared_ptr<syntax::SyntaxTree>, std::pair<SourceBuffer, bool>,
std::pair<const FileEntry*, std::error_code>>;
std::pair<const FileEntry*, std::error_code>,
std::pair<SourceBuffer, const UnitEntry*>>;

SourceLibrary* getOrAddLibrary(std::string_view name);
void addFilesInternal(std::string_view pattern, const std::filesystem::path& basePath,
bool isLibraryFile, const SourceLibrary* library, bool expandEnvVars);
bool isLibraryFile, const SourceLibrary* library, const UnitEntry* unit,
bool expandEnvVars);
void addLibraryMapsInternal(std::string_view pattern, const std::filesystem::path& basePath,
const Bag& optionBag, bool expandEnvVars,
flat_hash_set<std::filesystem::path>& seenMaps);
Expand All @@ -186,6 +214,7 @@ class SLANG_EXPORT SourceLoader {
std::vector<FileEntry> fileEntries;
flat_hash_map<std::filesystem::path, size_t> fileIndex;
flat_hash_map<std::string, std::unique_ptr<SourceLibrary>> libraries;
std::deque<UnitEntry> unitEntries;
std::vector<std::filesystem::path> searchDirectories;
std::vector<std::filesystem::path> searchExtensions;
flat_hash_set<std::string_view> uniqueExtensions;
Expand Down
91 changes: 80 additions & 11 deletions source/driver/Driver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,17 @@ void Driver::addStandardArgs() {
cmdLine.add("-j,--threads", options.numThreads,
"The number of threads to use to parallelize parsing", "<count>");

cmdLine.add(
"-C",
[this](std::string_view value) {
processCommandFiles(value, /* makeRelative */ true, /* separateUnit */ true);
return "";
},
"One or more files containing independent compilation unit listings. "
"The files accept a subset of options that pertain specifically to parsing "
"that unit and optionally including it in a library.",
"<file-pattern>[,...]", CommandLineFlags::CommaList);

// Compilation
cmdLine.add("--max-hierarchy-depth", options.maxInstanceDepth,
"Maximum depth of the design hierarchy", "<depth>");
Expand Down Expand Up @@ -289,7 +300,7 @@ void Driver::addStandardArgs() {
cmdLine.add(
"-f",
[this](std::string_view value) {
processCommandFiles(value, /* makeRelative */ false);
processCommandFiles(value, /* makeRelative */ false, /* separateUnit */ false);
return "";
},
"One or more command files containing additional program options. "
Expand All @@ -299,7 +310,7 @@ void Driver::addStandardArgs() {
cmdLine.add(
"-F",
[this](std::string_view value) {
processCommandFiles(value, /* makeRelative */ true);
processCommandFiles(value, /* makeRelative */ true, /* separateUnit */ false);
return "";
},
"One or more command files containing additional program options. "
Expand All @@ -317,7 +328,7 @@ void Driver::addStandardArgs() {
return !anyFailedLoads;
}

bool Driver::processCommandFiles(std::string_view pattern, bool makeRelative) {
bool Driver::processCommandFiles(std::string_view pattern, bool makeRelative, bool separateUnit) {
auto onError = [this](const auto& name, std::error_code ec) {
printError(fmt::format("command file '{}': {}", name, ec.message()));
anyFailedLoads = true;
Expand Down Expand Up @@ -349,17 +360,22 @@ bool Driver::processCommandFiles(std::string_view pattern, bool makeRelative) {
fs::current_path(path.parent_path(), ec);
}

CommandLine::ParseOptions parseOpts;
parseOpts.expandEnvVars = true;
parseOpts.ignoreProgramName = true;
parseOpts.supportComments = true;
parseOpts.ignoreDuplicates = true;

SLANG_ASSERT(!buffer.empty());
buffer.pop_back();

std::string_view argStr(buffer.data(), buffer.size());
const bool result = parseCommandLine(argStr, parseOpts);

bool result;
if (separateUnit) {
result = parseUnitListing(argStr);
}
else {
CommandLine::ParseOptions parseOpts;
parseOpts.expandEnvVars = true;
parseOpts.ignoreProgramName = true;
parseOpts.supportComments = true;
parseOpts.ignoreDuplicates = true;
result = parseCommandLine(argStr, parseOpts);
}

if (makeRelative)
fs::current_path(currPath, ec);
Expand Down Expand Up @@ -775,6 +791,59 @@ bool Driver::reportCompilation(Compilation& compilation, bool quiet) {
return succeeded;
}

bool Driver::parseUnitListing(std::string_view text) {
CommandLine unitCmdLine;
std::vector<std::string> includes;
unitCmdLine.add("-I,--include-directory,+incdir", includes, "", "",
CommandLineFlags::CommaList);

std::vector<std::string> defines;
unitCmdLine.add("-D,--define-macro,+define", defines, "");

std::optional<std::string> libraryName;
unitCmdLine.add("--library", libraryName, "");

unitCmdLine.add(
"-C",
[this](std::string_view value) {
processCommandFiles(value, /* makeRelative */ true, /* separateUnit */ true);
return "";
},
"", "", CommandLineFlags::CommaList);

std::vector<std::string> files;
unitCmdLine.setPositional(
[&](std::string_view value) {
if (!options.excludeExts.empty()) {
if (size_t extIndex = value.find_last_of('.'); extIndex != std::string_view::npos) {
if (options.excludeExts.count(std::string(value.substr(extIndex + 1))))
return "";
}
}

files.push_back(std::string(value));
return "";
},
"");

CommandLine::ParseOptions parseOpts;
parseOpts.expandEnvVars = true;
parseOpts.ignoreProgramName = true;
parseOpts.supportComments = true;
parseOpts.ignoreDuplicates = true;

if (!unitCmdLine.parse(text, parseOpts)) {
for (auto& err : unitCmdLine.getErrors())
OS::printE(fmt::format("{}\n", err));
return false;
}

sourceLoader.addSeparateUnit(files, std::move(includes), std::move(defines),
std::move(libraryName).value_or(std::string()));

return true;
}

void Driver::addLibraryFiles(std::string_view pattern) {
// Parse the pattern; there's an optional leading library name
// followed by an equals sign. If not there, we use the default
Expand Down
Loading

0 comments on commit 30d2d67

Please sign in to comment.