//===--- SourceCode.h - Manipulating source code as strings -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "SourceCode.h" #include "Logger.h" #include "clang/AST/ASTContext.h" #include "clang/Basic/SourceManager.h" #include "clang/Lex/Lexer.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" #include "llvm/Support/Path.h" namespace clang { namespace clangd { using namespace llvm; // Here be dragons. LSP positions use columns measured in *UTF-16 code units*! // Clangd uses UTF-8 and byte-offsets internally, so conversion is nontrivial. // Iterates over unicode codepoints in the (UTF-8) string. For each, // invokes CB(UTF-8 length, UTF-16 length), and breaks if it returns true. // Returns true if CB returned true, false if we hit the end of string. template static bool iterateCodepoints(StringRef U8, const Callback &CB) { for (size_t I = 0; I < U8.size();) { unsigned char C = static_cast(U8[I]); if (LLVM_LIKELY(!(C & 0x80))) { // ASCII character. if (CB(1, 1)) return true; ++I; continue; } // This convenient property of UTF-8 holds for all non-ASCII characters. size_t UTF8Length = countLeadingOnes(C); // 0xxx is ASCII, handled above. 10xxx is a trailing byte, invalid here. // 11111xxx is not valid UTF-8 at all. Assert because it's probably our bug. assert((UTF8Length >= 2 && UTF8Length <= 4) && "Invalid UTF-8, or transcoding bug?"); I += UTF8Length; // Skip over all trailing bytes. // A codepoint takes two UTF-16 code unit if it's astral (outside BMP). // Astral codepoints are encoded as 4 bytes in UTF-8 (11110xxx ...) if (CB(UTF8Length, UTF8Length == 4 ? 2 : 1)) return true; } return false; } // Returns the offset into the string that matches \p Units UTF-16 code units. // Conceptually, this converts to UTF-16, truncates to CodeUnits, converts back // to UTF-8, and returns the length in bytes. static size_t measureUTF16(StringRef U8, int U16Units, bool &Valid) { size_t Result = 0; Valid = U16Units == 0 || iterateCodepoints(U8, [&](int U8Len, int U16Len) { Result += U8Len; U16Units -= U16Len; return U16Units <= 0; }); if (U16Units < 0) // Offset was into the middle of a surrogate pair. Valid = false; // Don't return an out-of-range index if we overran. return std::min(Result, U8.size()); } // Counts the number of UTF-16 code units needed to represent a string. // Like most strings in clangd, the input is UTF-8 encoded. static size_t utf16Len(StringRef U8) { // A codepoint takes two UTF-16 code unit if it's astral (outside BMP). // Astral codepoints are encoded as 4 bytes in UTF-8, starting with 11110xxx. size_t Count = 0; iterateCodepoints(U8, [&](int U8Len, int U16Len) { Count += U16Len; return false; }); return Count; } llvm::Expected positionToOffset(StringRef Code, Position P, bool AllowColumnsBeyondLineLength) { if (P.line < 0) return llvm::make_error( llvm::formatv("Line value can't be negative ({0})", P.line), llvm::errc::invalid_argument); if (P.character < 0) return llvm::make_error( llvm::formatv("Character value can't be negative ({0})", P.character), llvm::errc::invalid_argument); size_t StartOfLine = 0; for (int I = 0; I != P.line; ++I) { size_t NextNL = Code.find('\n', StartOfLine); if (NextNL == StringRef::npos) return llvm::make_error( llvm::formatv("Line value is out of range ({0})", P.line), llvm::errc::invalid_argument); StartOfLine = NextNL + 1; } size_t NextNL = Code.find('\n', StartOfLine); if (NextNL == StringRef::npos) NextNL = Code.size(); bool Valid; size_t ByteOffsetInLine = measureUTF16( Code.substr(StartOfLine, NextNL - StartOfLine), P.character, Valid); if (!Valid && !AllowColumnsBeyondLineLength) return llvm::make_error( llvm::formatv("UTF-16 offset {0} is invalid for line {1}", P.character, P.line), llvm::errc::invalid_argument); return StartOfLine + ByteOffsetInLine; } Position offsetToPosition(StringRef Code, size_t Offset) { Offset = std::min(Code.size(), Offset); StringRef Before = Code.substr(0, Offset); int Lines = Before.count('\n'); size_t PrevNL = Before.rfind('\n'); size_t StartOfLine = (PrevNL == StringRef::npos) ? 0 : (PrevNL + 1); Position Pos; Pos.line = Lines; Pos.character = utf16Len(Before.substr(StartOfLine)); return Pos; } Position sourceLocToPosition(const SourceManager &SM, SourceLocation Loc) { // We use the SourceManager's line tables, but its column number is in bytes. FileID FID; unsigned Offset; std::tie(FID, Offset) = SM.getDecomposedSpellingLoc(Loc); Position P; P.line = static_cast(SM.getLineNumber(FID, Offset)) - 1; bool Invalid = false; StringRef Code = SM.getBufferData(FID, &Invalid); if (!Invalid) { auto ColumnInBytes = SM.getColumnNumber(FID, Offset) - 1; auto LineSoFar = Code.substr(Offset - ColumnInBytes, ColumnInBytes); P.character = utf16Len(LineSoFar); } return P; } Range halfOpenToRange(const SourceManager &SM, CharSourceRange R) { // Clang is 1-based, LSP uses 0-based indexes. Position Begin = sourceLocToPosition(SM, R.getBegin()); Position End = sourceLocToPosition(SM, R.getEnd()); return {Begin, End}; } std::pair offsetToClangLineColumn(StringRef Code, size_t Offset) { Offset = std::min(Code.size(), Offset); StringRef Before = Code.substr(0, Offset); int Lines = Before.count('\n'); size_t PrevNL = Before.rfind('\n'); size_t StartOfLine = (PrevNL == StringRef::npos) ? 0 : (PrevNL + 1); return {Lines + 1, Offset - StartOfLine + 1}; } std::pair splitQualifiedName(llvm::StringRef QName) { size_t Pos = QName.rfind("::"); if (Pos == llvm::StringRef::npos) return {StringRef(), QName}; return {QName.substr(0, Pos + 2), QName.substr(Pos + 2)}; } TextEdit replacementToEdit(StringRef Code, const tooling::Replacement &R) { Range ReplacementRange = { offsetToPosition(Code, R.getOffset()), offsetToPosition(Code, R.getOffset() + R.getLength())}; return {ReplacementRange, R.getReplacementText()}; } std::vector replacementsToEdits(StringRef Code, const tooling::Replacements &Repls) { std::vector Edits; for (const auto &R : Repls) Edits.push_back(replacementToEdit(Code, R)); return Edits; } llvm::Optional getRealPath(const FileEntry *F, const SourceManager &SourceMgr) { // Ideally, we get the real path from the FileEntry object. SmallString<128> FilePath = F->tryGetRealPathName(); if (!FilePath.empty()) { return FilePath.str().str(); } // Otherwise, we try to compute ourselves. vlog("FileEntry for {0} did not contain the real path.", F->getName()); llvm::SmallString<128> Path = F->getName(); if (!llvm::sys::path::is_absolute(Path)) { if (!SourceMgr.getFileManager().makeAbsolutePath(Path)) { log("Could not turn relative path to absolute: {0}", Path); return llvm::None; } } llvm::SmallString<128> RealPath; if (SourceMgr.getFileManager().getVirtualFileSystem()->getRealPath( Path, RealPath)) { log("Could not compute real path: {0}", Path); return Path.str().str(); } return RealPath.str().str(); } TextEdit toTextEdit(const FixItHint &FixIt, const SourceManager &M, const LangOptions &L) { TextEdit Result; Result.range = halfOpenToRange(M, Lexer::makeFileCharRange(FixIt.RemoveRange, M, L)); Result.newText = FixIt.CodeToInsert; return Result; } } // namespace clangd } // namespace clang