Skip to content

Commit 9bd1a6d

Browse files
SunrisepeakSunrisepeak
authored
feat: Clang module pipeline parity with GCC (#34)
BmiTraits abstraction for GCC/Clang BMI path differences, bmi_cache neutral naming (gcmFiles→bmiFiles), clang-scan-deps integration for per-file dyndep, and multi-module Clang E2E test.
1 parent 77ffe2d commit 9bd1a6d

12 files changed

Lines changed: 753 additions & 77 deletions

‎.agents/docs/2026-05-15-clang-parity-and-toolchain-abstraction.md‎

Lines changed: 465 additions & 0 deletions
Large diffs are not rendered by default.

‎.agents/docs/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# 开发/方案文档目录

‎src/bmi_cache.cppm‎

Lines changed: 28 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
//
33
// Layout (per docs/26-bmi-cache.md):
44
// $MCPP_HOME/bmi/<fingerprint>/deps/<indexName>/<pkgName>@<version>/
5-
// gcm.cache/<module>.gcm
5+
// {gcm,pcm}.cache/<module>.{gcm,pcm}
66
// obj/<file>.m.o + <file>.o
77
// manifest.txt (sentinel + file list)
88
//
@@ -31,20 +31,22 @@ struct CacheKey {
3131
std::string indexName; // "mcpplibs" / "xim" / ...
3232
std::string packageName; // "mcpplibs.cmdline"
3333
std::string version; // "0.0.1"
34+
std::string bmiDirName = "gcm.cache"; // "gcm.cache" | "pcm.cache"
35+
std::string manifestTag = "gcm"; // "gcm" | "pcm"
3436

3537
std::filesystem::path dir() const {
3638
return mcppHome / "bmi" / fingerprint / "deps"
3739
/ indexName / std::format("{}@{}", packageName, version);
3840
}
3941

4042
std::filesystem::path manifestFile() const { return dir() / "manifest.txt"; }
41-
std::filesystem::path gcmDir() const { return dir() / "gcm.cache"; }
43+
std::filesystem::path bmiDir() const { return dir() / bmiDirName; }
4244
std::filesystem::path objDir() const { return dir() / "obj"; }
4345
};
4446

4547
// File names (basename only) that belong to one dep package's cache entry.
4648
struct DepArtifacts {
47-
std::vector<std::string> gcmFiles; // basenames in gcm.cache/
49+
std::vector<std::string> bmiFiles; // basenames in bmiDir/
4850
std::vector<std::string> objFiles; // basenames in obj/
4951
};
5052

@@ -55,14 +57,14 @@ bool is_cached(const CacheKey& key);
5557
std::expected<DepArtifacts, std::string>
5658
read_manifest(const CacheKey& key);
5759

58-
// Copy missing cached files into projectTarget/{gcm.cache,obj}. Existing
59-
// project outputs are left untouched: GCC BMIs may differ byte-for-byte between
60+
// Copy missing cached files into projectTarget/{bmiDirName,obj}. Existing
61+
// project outputs are left untouched: BMIs may differ byte-for-byte between
6062
// equivalent builds, and overwriting them would dirty downstream modules.
6163
std::expected<DepArtifacts, std::string>
6264
stage_into(const CacheKey& key,
6365
const std::filesystem::path& projectTargetDir);
6466

65-
// Copy fresh build outputs from projectTarget/{gcm.cache,obj} → cache dir
67+
// Copy fresh build outputs from projectTarget/{bmiDirName,obj} → cache dir
6668
// and write manifest.txt last (atomic-ish sentinel).
6769
std::expected<void, std::string>
6870
populate_from(const CacheKey& key,
@@ -91,9 +93,9 @@ bool copy_one(const std::filesystem::path& from,
9193
return !ec;
9294
}
9395

94-
std::string serialize_manifest(const DepArtifacts& a) {
96+
std::string serialize_manifest(std::string_view tag, const DepArtifacts& a) {
9597
std::string out = "# Auto-generated by mcpp bmi_cache. Do not edit.\n";
96-
for (auto& g : a.gcmFiles) out += std::format("gcm: {}\n", g);
98+
for (auto& g : a.bmiFiles) out += std::format("{}: {}\n", tag, g);
9799
for (auto& o : a.objFiles) out += std::format("obj: {}\n", o);
98100
return out;
99101
}
@@ -107,7 +109,8 @@ parse_manifest(const std::filesystem::path& p) {
107109
while (std::getline(is, line)) {
108110
while (!line.empty() && (line.back() == '\r' || line.back() == ' ')) line.pop_back();
109111
if (line.empty() || line[0] == '#') continue;
110-
if (line.starts_with("gcm: ")) a.gcmFiles.push_back(line.substr(5));
112+
if (line.starts_with("gcm: ")) a.bmiFiles.push_back(line.substr(5));
113+
else if (line.starts_with("pcm: ")) a.bmiFiles.push_back(line.substr(5));
111114
else if (line.starts_with("obj: ")) a.objFiles.push_back(line.substr(5));
112115
}
113116
return a;
@@ -121,8 +124,8 @@ bool is_cached(const CacheKey& key) {
121124
auto arts = parse_manifest(mf);
122125
if (!arts) return false;
123126
// Verify every listed file actually exists on disk.
124-
for (auto& g : arts->gcmFiles) {
125-
if (!std::filesystem::exists(key.gcmDir() / g)) return false;
127+
for (auto& g : arts->bmiFiles) {
128+
if (!std::filesystem::exists(key.bmiDir() / g)) return false;
126129
}
127130
for (auto& o : arts->objFiles) {
128131
if (!std::filesystem::exists(key.objDir() / o)) return false;
@@ -142,23 +145,23 @@ stage_into(const CacheKey& key,
142145
auto arts = parse_manifest(key.manifestFile());
143146
if (!arts) return std::unexpected(arts.error());
144147

145-
auto projectGcm = projectTargetDir / "gcm.cache";
148+
auto projectBmi = projectTargetDir / key.bmiDirName;
146149
auto projectObj = projectTargetDir / "obj";
147150
std::error_code ec;
148-
std::filesystem::create_directories(projectGcm, ec);
151+
std::filesystem::create_directories(projectBmi, ec);
149152
std::filesystem::create_directories(projectObj, ec);
150153

151-
for (auto& g : arts->gcmFiles) {
152-
auto from = key.gcmDir() / g;
153-
auto to = projectGcm / g;
154+
for (auto& g : arts->bmiFiles) {
155+
auto from = key.bmiDir() / g;
156+
auto to = projectBmi / g;
154157
if (std::filesystem::exists(to, ec)) {
155158
ec.clear();
156159
continue;
157160
}
158161
ec.clear();
159162
if (!copy_one(from, to, ec)) {
160163
return std::unexpected(std::format(
161-
"stage gcm '{}': {}", g, ec.message()));
164+
"stage bmi '{}': {}", g, ec.message()));
162165
}
163166
touch_now(to);
164167
}
@@ -221,25 +224,25 @@ populate_from(const CacheKey& key,
221224
~LockGuard() { release_lock(fd); }
222225
} guard{ lockFd };
223226

224-
auto cacheGcm = key.gcmDir();
227+
auto cacheBmi = key.bmiDir();
225228
auto cacheObj = key.objDir();
226229
std::error_code ec;
227-
std::filesystem::create_directories(cacheGcm, ec);
230+
std::filesystem::create_directories(cacheBmi, ec);
228231
std::filesystem::create_directories(cacheObj, ec);
229232

230-
auto projectGcm = projectTargetDir / "gcm.cache";
233+
auto projectBmi = projectTargetDir / key.bmiDirName;
231234
auto projectObj = projectTargetDir / "obj";
232235

233-
for (auto& g : arts.gcmFiles) {
234-
auto from = projectGcm / g;
235-
auto to = cacheGcm / g;
236+
for (auto& g : arts.bmiFiles) {
237+
auto from = projectBmi / g;
238+
auto to = cacheBmi / g;
236239
if (!std::filesystem::exists(from)) {
237240
return std::unexpected(std::format(
238241
"expected build output missing: {}", from.string()));
239242
}
240243
if (!copy_one(from, to, ec)) {
241244
return std::unexpected(std::format(
242-
"populate gcm '{}': {}", g, ec.message()));
245+
"populate bmi '{}': {}", g, ec.message()));
243246
}
244247
}
245248
for (auto& o : arts.objFiles) {
@@ -260,7 +263,7 @@ populate_from(const CacheKey& key,
260263
tmp += ".tmp";
261264
{
262265
std::ofstream os(tmp);
263-
os << serialize_manifest(arts);
266+
os << serialize_manifest(key.manifestTag, arts);
264267
}
265268
std::filesystem::rename(tmp, key.manifestFile(), ec);
266269
if (ec) {

‎src/build/flags.cppm‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,13 @@ CompileFlags compute_flags(const BuildPlan& plan) {
129129
if (isClang && !plan.stdBmiPath.empty()) {
130130
std_module_flag = " -fmodule-file=std=" + escape_path(staged_std_bmi_path(plan));
131131
}
132-
f.cxx = std::format("-std=c++23{}{}{}{}{}{}{}{}", module_flag, std_module_flag,
132+
auto traits = mcpp::toolchain::bmi_traits(plan.toolchain);
133+
std::string prebuilt_module_flag;
134+
if (traits.needsPrebuiltModulePath) {
135+
prebuilt_module_flag = std::format(" -fprebuilt-module-path={}", traits.bmiDir);
136+
}
137+
f.cxx = std::format("-std=c++23{}{}{}{}{}{}{}{}{}", module_flag, std_module_flag,
138+
prebuilt_module_flag,
133139
opt_flag, pic_flag, sysroot_flag, b_flag, include_flags, user_cxxflags);
134140
f.cc = std::format("-std={}{}{}{}{}{}{}", c_std, opt_flag, pic_flag, sysroot_flag, b_flag,
135141
include_flags, user_cflags);

‎src/build/ninja_backend.cppm‎

Lines changed: 53 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,13 @@ bool is_c_source(const std::filesystem::path& src) {
126126
} // namespace
127127

128128
std::string emit_ninja_string(const BuildPlan& plan) {
129-
bool dyndep = dyndep_mode_enabled()
130-
&& mcpp::toolchain::is_gcc(plan.toolchain);
129+
// dyndep requires P1689 scanning capability:
130+
// GCC: built-in -fdeps-format=p1689r5
131+
// Clang: external clang-scan-deps tool (same P1689 output format)
132+
bool has_scanner = mcpp::toolchain::is_gcc(plan.toolchain)
133+
|| !plan.scanDepsPath.empty();
134+
bool dyndep = dyndep_mode_enabled() && has_scanner;
135+
auto traits = mcpp::toolchain::bmi_traits(plan.toolchain);
131136
std::string out;
132137
auto append = [&](std::string s) { out += std::move(s); };
133138

@@ -162,6 +167,9 @@ std::string emit_ninja_string(const BuildPlan& plan) {
162167
}
163168
if (dyndep) {
164169
append(std::format("mcpp = {}\n", escape_ninja_path(mcpp_exe_path())));
170+
if (!plan.scanDepsPath.empty()) {
171+
append(std::format("scan_deps = {}\n", escape_ninja_path(plan.scanDepsPath)));
172+
}
165173
}
166174
append("\n");
167175

@@ -170,10 +178,12 @@ std::string emit_ninja_string(const BuildPlan& plan) {
170178
append(" description = STAGE $out\n\n");
171179

172180
// P1: per-file dyndep rule. Converts one .ddi → .dd independently.
173-
append("rule cxx_dyndep\n");
174-
append(" command = $mcpp dyndep --single --output $out $in\n");
175-
append(" description = DYNDEP $out\n");
176-
append(" restat = 1\n\n");
181+
append(std::format(
182+
"rule cxx_dyndep\n"
183+
" command = $mcpp dyndep --single --bmi-dir {} --bmi-ext {} --output $out $in\n"
184+
" description = DYNDEP $out\n"
185+
" restat = 1\n\n",
186+
traits.bmiDir, traits.bmiExt));
177187

178188
// P2: cxx_module preserves BMI timestamps when interface is unchanged.
179189
// GCC always updates the .gcm timestamp even if content is identical.
@@ -184,18 +194,20 @@ std::string emit_ninja_string(const BuildPlan& plan) {
184194
//
185195
// $bmi_out is set per build edge to the BMI path (gcm.cache/<module>.gcm).
186196
// If $bmi_out is empty (no module provided), we just compile normally.
197+
std::string module_output_flag = traits.needsExplicitModuleOutput
198+
? " -fmodule-output=$bmi_out" : "";
187199
append("rule cxx_module\n");
188-
append(" command = "
200+
append(std::format(" command = "
189201
"if [ -n \"$bmi_out\" ] && [ -f \"$bmi_out\" ]; then "
190202
"cp -p \"$bmi_out\" \"$bmi_out.bak\"; "
191203
"fi && "
192-
"$toolenv $cxx $cxxflags -c $in -o $out && "
204+
"$toolenv $cxx $cxxflags{} -c $in -o $out && "
193205
"if [ -n \"$bmi_out\" ] && [ -f \"$bmi_out.bak\" ] && "
194206
"cmp -s \"$bmi_out\" \"$bmi_out.bak\"; then "
195207
"mv \"$bmi_out.bak\" \"$bmi_out\"; "
196208
"else "
197209
"rm -f \"$bmi_out.bak\"; "
198-
"fi\n");
210+
"fi\n", module_output_flag));
199211
append(" description = MOD $out\n");
200212
if (dyndep)
201213
append(" restat = 1\n");
@@ -231,18 +243,31 @@ std::string emit_ninja_string(const BuildPlan& plan) {
231243

232244
if (dyndep) {
233245
// Scan rule: produce P1689 .ddi for one TU.
234-
// -E -M -MM -MF gives us the dep file; -fdeps-* gives us the .ddi.
246+
// GCC: built-in -fdeps-format=p1689r5 flags during preprocessing.
247+
// Clang: external clang-scan-deps tool with -format=p1689.
235248
append("rule cxx_scan\n");
236-
append(" command = $toolenv $cxx $cxxflags -fdeps-format=p1689r5 "
237-
"-fdeps-file=$out -fdeps-target=$compile_target "
238-
"-M -MM -MF $out.dep -E $in -o $compile_target\n");
249+
if (plan.scanDepsPath.empty()) {
250+
// GCC path: compiler-integrated P1689 scanning.
251+
append(" command = $toolenv $cxx $cxxflags -fmodules "
252+
"-fdeps-format=p1689r5 "
253+
"-fdeps-file=$out -fdeps-target=$compile_target "
254+
"-M -MM -MF $out.dep -E $in -o $compile_target\n");
255+
} else {
256+
// Clang path: clang-scan-deps produces P1689 JSON to stdout,
257+
// then we redirect to $out. The -- separator passes the full
258+
// compile command so clang-scan-deps knows the flags/sysroot.
259+
append(" command = $toolenv $scan_deps -format=p1689 -- "
260+
"$cxx $cxxflags -c $in -o $compile_target > $out\n");
261+
}
239262
append(" description = SCAN $out\n\n");
240263

241264
// Aggregate .ddi files into a Ninja dyndep file.
242-
append("rule cxx_collect\n");
243-
append(" command = $mcpp dyndep --output $out $in\n");
244-
append(" description = COLLECT $out\n");
245-
append(" restat = 1\n\n");
265+
append(std::format(
266+
"rule cxx_collect\n"
267+
" command = $mcpp dyndep --bmi-dir {} --bmi-ext {} --output $out $in\n"
268+
" description = COLLECT $out\n"
269+
" restat = 1\n\n",
270+
traits.bmiDir, traits.bmiExt));
246271
}
247272

248273
// Stage prebuilt std artifacts into the compiler-specific BMI cache.
@@ -257,11 +282,12 @@ std::string emit_ninja_string(const BuildPlan& plan) {
257282
escape_ninja_path(plan.stdObjectPath)));
258283
}
259284

260-
auto bmi_path = [](std::string_view name) {
261-
std::string s = "gcm.cache/";
285+
auto bmi_path = [&traits](std::string_view name) {
286+
std::string s(traits.bmiDir);
287+
s += '/';
262288
for (char c : name)
263289
s.push_back(c == ':' ? '-' : c);
264-
s += ".gcm";
290+
s += traits.bmiExt;
265291
return s;
266292
};
267293

@@ -360,12 +386,18 @@ std::string emit_ninja_string(const BuildPlan& plan) {
360386

361387
std::string out_line = "build " + escape_ninja_path(cu.object);
362388
if (cu.providesModule) {
363-
out_line += " " + bmi_path(*cu.providesModule);
389+
// Use implicit output (|) so $out only contains the .o file.
390+
// GCC writes BMI implicitly; Clang uses -fmodule-output=$bmi_out.
391+
out_line += " | " + bmi_path(*cu.providesModule);
364392
}
365393
out_line += std::format(" : {} {}", rule, escape_ninja_path(cu.source));
366394
if (!implicit.empty())
367395
out_line += " |" + implicit;
368396
out_line += "\n";
397+
// Clang needs $bmi_out to emit -fmodule-output=$bmi_out
398+
if (cu.providesModule) {
399+
out_line += " bmi_out = " + bmi_path(*cu.providesModule) + "\n";
400+
}
369401
append(std::move(out_line));
370402
}
371403
append("\n");

‎src/build/plan.cppm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ struct BuildPlan {
3737
std::filesystem::path outputDir; // target/<triple>/<fp>/
3838
std::filesystem::path stdBmiPath; // absolute path to prebuilt std.gcm
3939
std::filesystem::path stdObjectPath; // absolute path to prebuilt std.o
40+
std::filesystem::path scanDepsPath; // clang-scan-deps binary (Clang only)
4041

4142
std::vector<CompileUnit> compileUnits; // topologically sorted
4243
std::vector<LinkUnit> linkUnits;

0 commit comments

Comments
 (0)