Skip to content

Commit

Permalink
[rebranch] Temporarily revert "[Serialization] No transitive identifi…
Browse files Browse the repository at this point in the history 
…er change (llvm#92085)"

Temporarily reverts 2f2ea35 while we
fix Swift's ClangImporter to handle the 32 bit -> 64 bit ID change.
  • Loading branch information
@bnbarham
bnbarham committed Jul 4, 2024
1 parent 7914bae commit 6e41e1f
Show file tree
Hide file tree
Showing 9 changed files with 82 additions and 208 deletions.
2 changes: 1 addition & 1 deletion clang/include/clang/Lex/ExternalPreprocessorSource.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class ExternalPreprocessorSource {
/// Return the identifier associated with the given ID number.
///
/// The ID 0 is associated with the NULL identifier.
virtual IdentifierInfo *GetIdentifier(uint64_t ID) = 0;
virtual IdentifierInfo *GetIdentifier(unsigned ID) = 0;

/// Map a module ID to a module.
virtual Module *getModule(unsigned ModuleID) = 0;
Expand Down
2 changes: 1 addition & 1 deletion clang/include/clang/Serialization/ASTBitCodes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ const unsigned VERSION_MINOR = 1;
///
/// The ID numbers of identifiers are consecutive (in order of discovery)
/// and start at 1. 0 is reserved for NULL.
using IdentifierID = uint64_t;
using IdentifierID = uint32_t;

/// The number of predefined identifier IDs.
const unsigned int NUM_PREDEF_IDENT_IDS = 1;
Expand Down
19 changes: 11 additions & 8 deletions clang/include/clang/Serialization/ASTReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -677,6 +677,14 @@ class ASTReader
/// been loaded.
std::vector<IdentifierInfo *> IdentifiersLoaded;

using GlobalIdentifierMapType =
ContinuousRangeMap<serialization::IdentifierID, ModuleFile *, 4>;

/// Mapping from global identifier IDs to the module in which the
/// identifier resides along with the offset that should be added to the
/// global identifier ID to produce a local ID.
GlobalIdentifierMapType GlobalIdentifierMap;

/// A vector containing macros that have already been
/// loaded.
///
Expand Down Expand Up @@ -1548,11 +1556,6 @@ class ASTReader
/// Translate a \param GlobalDeclID to the index of DeclsLoaded array.
unsigned translateGlobalDeclIDToIndex(GlobalDeclID ID) const;

/// Translate an \param IdentifierID ID to the index of IdentifiersLoaded
/// array and the corresponding module file.
std::pair<ModuleFile *, unsigned>
translateIdentifierIDToIndex(serialization::IdentifierID ID) const;

/// Translate an \param TypeID ID to the index of TypesLoaded
/// array and the corresponding module file.
std::pair<ModuleFile *, unsigned>
Expand Down Expand Up @@ -2140,7 +2143,7 @@ class ASTReader
/// Load a selector from disk, registering its ID if it exists.
void LoadSelector(Selector Sel);

void SetIdentifierInfo(serialization::IdentifierID ID, IdentifierInfo *II);
void SetIdentifierInfo(unsigned ID, IdentifierInfo *II);
void SetGloballyVisibleDecls(IdentifierInfo *II,
const SmallVectorImpl<GlobalDeclID> &DeclIDs,
SmallVectorImpl<Decl *> *Decls = nullptr);
Expand All @@ -2167,10 +2170,10 @@ class ASTReader
return DecodeIdentifierInfo(ID);
}

IdentifierInfo *getLocalIdentifier(ModuleFile &M, uint64_t LocalID);
IdentifierInfo *getLocalIdentifier(ModuleFile &M, unsigned LocalID);

serialization::IdentifierID getGlobalIdentifierID(ModuleFile &M,
uint64_t LocalID);
unsigned LocalID);

void resolvePendingMacro(IdentifierInfo *II, const PendingMacroInfo &PMInfo);

Expand Down
3 changes: 3 additions & 0 deletions clang/include/clang/Serialization/ModuleFile.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,9 @@ class ModuleFile {
/// Base identifier ID for identifiers local to this module.
serialization::IdentifierID BaseIdentifierID = 0;

/// Remapping table for identifier IDs in this module.
ContinuousRangeMap<uint32_t, int, 2> IdentifierRemap;

/// Actual data for the on-disk hash table of identifiers.
///
/// This pointer points into a memory buffer, where the on-disk hash
Expand Down
96 changes: 45 additions & 51 deletions clang/lib/Serialization/ASTReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -969,7 +969,7 @@ ASTSelectorLookupTrait::ReadKey(const unsigned char* d, unsigned) {
SelectorTable &SelTable = Reader.getContext().Selectors;
unsigned N = endian::readNext<uint16_t, llvm::endianness::little>(d);
const IdentifierInfo *FirstII = Reader.getLocalIdentifier(
F, endian::readNext<IdentifierID, llvm::endianness::little>(d));
F, endian::readNext<uint32_t, llvm::endianness::little>(d));
if (N == 0)
return SelTable.getNullarySelector(FirstII);
else if (N == 1)
Expand All @@ -979,7 +979,7 @@ ASTSelectorLookupTrait::ReadKey(const unsigned char* d, unsigned) {
Args.push_back(FirstII);
for (unsigned I = 1; I != N; ++I)
Args.push_back(Reader.getLocalIdentifier(
F, endian::readNext<IdentifierID, llvm::endianness::little>(d)));
F, endian::readNext<uint32_t, llvm::endianness::little>(d)));

return SelTable.getSelector(N, Args.data());
}
Expand Down Expand Up @@ -1062,8 +1062,7 @@ static bool readBit(unsigned &Bits) {
IdentifierID ASTIdentifierLookupTrait::ReadIdentifierID(const unsigned char *d) {
using namespace llvm::support;

IdentifierID RawID =
endian::readNext<IdentifierID, llvm::endianness::little>(d);
unsigned RawID = endian::readNext<uint32_t, llvm::endianness::little>(d);
return Reader.getGlobalIdentifierID(F, RawID >> 1);
}

Expand All @@ -1081,12 +1080,9 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
unsigned DataLen) {
using namespace llvm::support;

IdentifierID RawID =
endian::readNext<IdentifierID, llvm::endianness::little>(d);
unsigned RawID = endian::readNext<uint32_t, llvm::endianness::little>(d);
bool IsInteresting = RawID & 0x01;

DataLen -= sizeof(IdentifierID);

// Wipe out the "is interesting" bit.
RawID = RawID >> 1;

Expand Down Expand Up @@ -1117,7 +1113,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
bool HadMacroDefinition = readBit(Bits);

assert(Bits == 0 && "Extra bits in the identifier?");
DataLen -= sizeof(uint16_t) * 2;
DataLen -= 8;

// Set or check the various bits in the IdentifierInfo structure.
// Token IDs are read-only.
Expand Down Expand Up @@ -1244,7 +1240,7 @@ ASTDeclContextNameLookupTrait::ReadKey(const unsigned char *d, unsigned) {
case DeclarationName::CXXLiteralOperatorName:
case DeclarationName::CXXDeductionGuideName:
Data = (uint64_t)Reader.getLocalIdentifier(
F, endian::readNext<IdentifierID, llvm::endianness::little>(d));
F, endian::readNext<uint32_t, llvm::endianness::little>(d));
break;
case DeclarationName::ObjCZeroArgSelector:
case DeclarationName::ObjCOneArgSelector:
Expand Down Expand Up @@ -2115,7 +2111,7 @@ HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d,
HFI.DirInfo = (Flags >> 1) & 0x07;
HFI.IndexHeaderMapHeader = Flags & 0x01;
HFI.LazyControllingMacro = Reader.getGlobalIdentifierID(
M, endian::readNext<IdentifierID, llvm::endianness::little>(d));
M, endian::readNext<uint32_t, llvm::endianness::little>(d));
if (unsigned FrameworkOffset =
endian::readNext<uint32_t, llvm::endianness::little>(d)) {
// The framework offset is 1 greater than the actual offset,
Expand Down Expand Up @@ -3513,11 +3509,24 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
"duplicate IDENTIFIER_OFFSET record in AST file");
F.IdentifierOffsets = (const uint32_t *)Blob.data();
F.LocalNumIdentifiers = Record[0];
unsigned LocalBaseIdentifierID = Record[1];
F.BaseIdentifierID = getTotalNumIdentifiers();

if (F.LocalNumIdentifiers > 0)
if (F.LocalNumIdentifiers > 0) {
// Introduce the global -> local mapping for identifiers within this
// module.
GlobalIdentifierMap.insert(std::make_pair(getTotalNumIdentifiers() + 1,
&F));

// Introduce the local -> global mapping for identifiers within this
// module.
F.IdentifierRemap.insertOrReplace(
std::make_pair(LocalBaseIdentifierID,
F.BaseIdentifierID - LocalBaseIdentifierID));

IdentifiersLoaded.resize(IdentifiersLoaded.size()
+ F.LocalNumIdentifiers);
}
break;
}

Expand Down Expand Up @@ -4104,6 +4113,7 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
F.ModuleOffsetMap = StringRef();

using RemapBuilder = ContinuousRangeMap<uint32_t, int, 2>::Builder;
RemapBuilder IdentifierRemap(F.IdentifierRemap);
RemapBuilder MacroRemap(F.MacroRemap);
RemapBuilder PreprocessedEntityRemap(F.PreprocessedEntityRemap);
RemapBuilder SubmoduleRemap(F.SubmoduleRemap);
Expand Down Expand Up @@ -4135,6 +4145,8 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {

ImportedModuleVector.push_back(OM);

uint32_t IdentifierIDOffset =
endian::readNext<uint32_t, llvm::endianness::little>(Data);
uint32_t MacroIDOffset =
endian::readNext<uint32_t, llvm::endianness::little>(Data);
uint32_t PreprocessedEntityIDOffset =
Expand All @@ -4152,6 +4164,7 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
static_cast<int>(BaseOffset - Offset)));
};

mapOffset(IdentifierIDOffset, OM->BaseIdentifierID, IdentifierRemap);
mapOffset(MacroIDOffset, OM->BaseMacroID, MacroRemap);
mapOffset(PreprocessedEntityIDOffset, OM->BasePreprocessedEntityID,
PreprocessedEntityRemap);
Expand Down Expand Up @@ -8289,6 +8302,7 @@ LLVM_DUMP_METHOD void ASTReader::dump() {
llvm::errs() << "*** PCH/ModuleFile Remappings:\n";
dumpModuleIDMap("Global bit offset map", GlobalBitOffsetsMap);
dumpModuleIDMap("Global source location entry map", GlobalSLocEntryMap);
dumpModuleIDMap("Global identifier map", GlobalIdentifierMap);
dumpModuleIDMap("Global macro map", GlobalMacroMap);
dumpModuleIDMap("Global submodule map", GlobalSubmoduleMap);
dumpModuleIDMap("Global selector map", GlobalSelectorMap);
Expand Down Expand Up @@ -8928,9 +8942,8 @@ void ASTReader::LoadSelector(Selector Sel) {

void ASTReader::SetIdentifierInfo(IdentifierID ID, IdentifierInfo *II) {
assert(ID && "Non-zero identifier ID required");
unsigned Index = translateIdentifierIDToIndex(ID).second;
assert(Index < IdentifiersLoaded.size() && "identifier ID out of range");
IdentifiersLoaded[Index] = II;
assert(ID <= IdentifiersLoaded.size() && "identifier ID out of range");
IdentifiersLoaded[ID - 1] = II;
if (DeserializationListener)
DeserializationListener->IdentifierRead(ID, II);
}
Expand Down Expand Up @@ -8983,22 +8996,6 @@ void ASTReader::SetGloballyVisibleDecls(
}
}

std::pair<ModuleFile *, unsigned>
ASTReader::translateIdentifierIDToIndex(IdentifierID ID) const {
if (ID == 0)
return {nullptr, 0};

unsigned ModuleFileIndex = ID >> 32;
unsigned LocalID = ID & llvm::maskTrailingOnes<IdentifierID>(32);

assert(ModuleFileIndex && "not translating loaded IdentifierID?");
assert(getModuleManager().size() > ModuleFileIndex - 1);

ModuleFile &MF = getModuleManager()[ModuleFileIndex - 1];
assert(LocalID < MF.LocalNumIdentifiers);
return {&MF, MF.BaseIdentifierID + LocalID};
}

IdentifierInfo *ASTReader::DecodeIdentifierInfo(IdentifierID ID) {
if (ID == 0)
return nullptr;
Expand All @@ -9008,48 +9005,45 @@ IdentifierInfo *ASTReader::DecodeIdentifierInfo(IdentifierID ID) {
return nullptr;
}

auto [M, Index] = translateIdentifierIDToIndex(ID);
if (!IdentifiersLoaded[Index]) {
assert(M != nullptr && "Untranslated Identifier ID?");
assert(Index >= M->BaseIdentifierID);
unsigned LocalIndex = Index - M->BaseIdentifierID;
ID -= 1;
if (!IdentifiersLoaded[ID]) {
GlobalIdentifierMapType::iterator I = GlobalIdentifierMap.find(ID + 1);
assert(I != GlobalIdentifierMap.end() && "Corrupted global identifier map");
ModuleFile *M = I->second;
unsigned Index = ID - M->BaseIdentifierID;
const unsigned char *Data =
M->IdentifierTableData + M->IdentifierOffsets[LocalIndex];
M->IdentifierTableData + M->IdentifierOffsets[Index];

ASTIdentifierLookupTrait Trait(*this, *M);
auto KeyDataLen = Trait.ReadKeyDataLength(Data);
auto Key = Trait.ReadKey(Data, KeyDataLen.first);
auto &II = PP.getIdentifierTable().get(Key);
IdentifiersLoaded[Index] = &II;
IdentifiersLoaded[ID] = &II;
markIdentifierFromAST(*this, II);
if (DeserializationListener)
DeserializationListener->IdentifierRead(ID, &II);
DeserializationListener->IdentifierRead(ID + 1, &II);
}

return IdentifiersLoaded[Index];
return IdentifiersLoaded[ID];
}

IdentifierInfo *ASTReader::getLocalIdentifier(ModuleFile &M, uint64_t LocalID) {
IdentifierInfo *ASTReader::getLocalIdentifier(ModuleFile &M, unsigned LocalID) {
return DecodeIdentifierInfo(getGlobalIdentifierID(M, LocalID));
}

IdentifierID ASTReader::getGlobalIdentifierID(ModuleFile &M, uint64_t LocalID) {
IdentifierID ASTReader::getGlobalIdentifierID(ModuleFile &M, unsigned LocalID) {
if (LocalID < NUM_PREDEF_IDENT_IDS)
return LocalID;

if (!M.ModuleOffsetMap.empty())
ReadModuleOffsetMap(M);

unsigned ModuleFileIndex = LocalID >> 32;
LocalID &= llvm::maskTrailingOnes<IdentifierID>(32);
ModuleFile *MF =
ModuleFileIndex ? M.TransitiveImports[ModuleFileIndex - 1] : &M;
assert(MF && "malformed identifier ID encoding?");

if (!ModuleFileIndex)
LocalID -= NUM_PREDEF_IDENT_IDS;
ContinuousRangeMap<uint32_t, int, 2>::iterator I
= M.IdentifierRemap.find(LocalID - NUM_PREDEF_IDENT_IDS);
assert(I != M.IdentifierRemap.end()
&& "Invalid index into identifier index remap");

return ((IdentifierID)(MF->Index + 1) << 32) | LocalID;
return LocalID + I->second;
}

MacroInfo *ASTReader::getMacro(MacroID ID) {
Expand Down
Loading

0 comments on commit 6e41e1f

Please sign in to comment.