Fix #14113 (Slow analysis: microchip xc16 code)

.github/workflows/selfcheck.yml

@@ -121,7 +121,7 @@ jobs:
 
       - name: Self check (unusedFunction / no test / no gui)
         run: |
-          supprs="--suppress=unusedFunction:lib/errorlogger.h:197 --suppress=unusedFunction:lib/importproject.cpp:1671 --suppress=unusedFunction:lib/importproject.cpp:1695"
+          supprs="--suppress=unusedFunction:lib/errorlogger.h:198 --suppress=unusedFunction:lib/importproject.cpp:1671 --suppress=unusedFunction:lib/importproject.cpp:1695"
           ./cppcheck -q --template=selfcheck --error-exitcode=1 --library=cppcheck-lib -D__CPPCHECK__ -D__GNUC__ --enable=unusedFunction,information --exception-handling -rp=. --project=cmake.output.notest_nogui/compile_commands.json --suppressions-list=.selfcheck_unused_suppressions --inline-suppr $supprs
         env:
           DISABLE_VALUEFLOW: 1

.selfcheck_suppressions

@@ -62,6 +62,8 @@ templateInstantiation:test/testutils.cpp
 
 naming-varname:externals/simplecpp/simplecpp.h
 naming-privateMemberVariable:externals/simplecpp/simplecpp.h
+# false positive; lambda captures its owner
+danglingLifetime:externals/simplecpp/simplecpp.h:505
 
 # TODO: these warnings need to be addressed upstream
 uninitMemberVar:externals/tinyxml2/tinyxml2.h

lib/cppcheck.cpp

@@ -61,6 +61,7 @@
 #include <exception> // IWYU pragma: keep
 #include <fstream>
 #include <iostream>
+#include <iterator>
 #include <map>
 #include <new>
 #include <set>
@@ -100,9 +101,9 @@ class CppCheck::CppCheckLogger : public ErrorLogger
         closePlist();
     }
 
-    void setRemarkComments(std::vector<RemarkComment> remarkComments)
+    void addRemarkComments(const std::vector<RemarkComment> &remarkComments)
     {
-        mRemarkComments = std::move(remarkComments);
+        mRemarkComments.insert(mRemarkComments.end(), remarkComments.begin(), remarkComments.end());
     }
 
     void setLocationMacros(const Token* startTok, const std::vector<std::string>& files)
@@ -124,17 +125,25 @@ class CppCheck::CppCheckLogger : public ErrorLogger
         mErrorList.clear();
     }
 
-    void openPlist(const std::string& filename, const std::vector<std::string>& files)
+    void openPlist(const std::string& filename)
     {
         mPlistFile.open(filename);
-        mPlistFile << ErrorLogger::plistHeader(version(), files);
+        mPlistFile << ErrorLogger::plistHeader(version());
+    }
+
+    void setPlistFilenames(std::vector<std::string> files)
+    {
+        if (mPlistFile.is_open()) {
+            mPlistFilenames = std::move(files);
+        }
     }
 
     void closePlist()
     {
         if (mPlistFile.is_open()) {
-            mPlistFile << ErrorLogger::plistFooter();
+            mPlistFile << ErrorLogger::plistFooter(mPlistFilenames);
             mPlistFile.close();
+            mPlistFilenames.clear();
         }
     }
 
@@ -282,6 +291,7 @@ class CppCheck::CppCheckLogger : public ErrorLogger
     std::map<Location, std::set<std::string>> mLocationMacros; // What macros are used on a location?
 
     std::ofstream mPlistFile;
+    std::vector<std::string> mPlistFilenames;
 
     unsigned int mExitCode{};
 
@@ -898,7 +908,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
     return checkInternal(file, cfgname, f);
 }
 
-void CppCheck::checkPlistOutput(const FileWithDetails& file, const std::vector<std::string>& files)
+void CppCheck::checkPlistOutput(const FileWithDetails& file)
 {
     if (!mSettings.plistOutput.empty()) {
         const bool slashFound = file.spath().find('/') != std::string::npos;
@@ -909,7 +919,7 @@ void CppCheck::checkPlistOutput(const FileWithDetails& file, const std::vector<s
         // the hash is added to handle when files in different folders have the same name
         const std::size_t fileNameHash = std::hash<std::string> {}(file.spath());
         filename = mSettings.plistOutput + noSuffixFilename + "_" + std::to_string(fileNameHash) + ".plist";
-        mLogger->openPlist(filename, files);
+        mLogger->openPlist(filename);
     }
 }
 
@@ -1000,24 +1010,16 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
         if (preprocessor.reportOutput(outputList, true))
             return mLogger->exitcode();
 
-        if (!preprocessor.loadFiles(files))
-            return mLogger->exitcode();
-
-        checkPlistOutput(file, files);
+        checkPlistOutput(file);
 
-        std::string dumpProlog;
+        std::string dumpFooter;
         if (mSettings.dump || !mSettings.addons.empty()) {
-            dumpProlog += getDumpFileContentsRawTokens(files, tokens1);
+            dumpFooter += getDumpFileContentsRawTokensFooter(tokens1);
         }
 
         // Parse comments and then remove them
-        mLogger->setRemarkComments(preprocessor.getRemarkComments());
+        mLogger->addRemarkComments(preprocessor.getRemarkComments());
         preprocessor.inlineSuppressions(mSuppressions.nomsg);
-        if (mSettings.dump || !mSettings.addons.empty()) {
-            std::ostringstream oss;
-            mSuppressions.nomsg.dump(oss);
-            dumpProlog += oss.str();
-        }
         preprocessor.removeComments();
 
         if (!mSettings.buildDir.empty()) {
@@ -1040,19 +1042,45 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
         }
 
         // Get directives
-        std::list<Directive> directives = preprocessor.createDirectives();
+        std::list<Directive> directives;
+        preprocessor.createDirectives(directives);
         preprocessor.simplifyPragmaAsm();
 
+        std::set<std::string> configurations;
+        std::set<std::string> configDefines = { "__cplusplus" };
+
+        // Insert library defines
+        const auto getDefineName = [](const std::string &defineString) {
+            return defineString.substr(0, defineString.find_first_of("( "));
+        };
+        std::transform(mSettings.library.defines().begin(),
+                       mSettings.library.defines().end(),
+                       std::inserter(configDefines, configDefines.end()),
+                       getDefineName);
+
+        preprocessor.setLoadCallback([&](simplecpp::FileData &data) {
+            // Do preprocessing on included file
+            mLogger->addRemarkComments(preprocessor.getRemarkComments(data.tokens));
+            preprocessor.inlineSuppressions(data.tokens, mSuppressions.nomsg);
+            Preprocessor::removeComments(data.tokens);
+            Preprocessor::createDirectives(data.tokens, directives);
+            Preprocessor::simplifyPragmaAsm(data.tokens);
+            // Discover new configurations from included file
+            if (configurations.size() < maxConfigs)
+                preprocessor.getConfigs(data.filename, data.tokens, configDefines, configurations);
+        });
+
         preprocessor.setPlatformInfo();
 
         // Get configurations..
-        std::set<std::string> configurations;
         if (maxConfigs > 1) {
             Timer::run("Preprocessor::getConfigs", mTimerResults, [&]() {
-                configurations = preprocessor.getConfigs();
+                configurations = { "" };
+                preprocessor.getConfigs(configDefines, configurations);
+                preprocessor.loadFiles(files);
             });
         } else {
-            configurations.insert(mSettings.userDefines);
+            configurations = { mSettings.userDefines };
         }
 
         if (mSettings.checkConfiguration) {
@@ -1089,7 +1117,6 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
         createDumpFile(mSettings, file, fdump, dumpFile);
         if (fdump.is_open()) {
             fdump << getLibraryDumpData();
-            fdump << dumpProlog;
             if (!mSettings.dump)
                 filesDeleter.addFile(dumpFile);
         }
@@ -1259,12 +1286,20 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
         }
 
         // TODO: will not be closed if we encountered an exception
-        // dumped all configs, close root </dumps> element now
         if (fdump.is_open()) {
+            // dump all filenames, raw tokens, suppressions
+            std::string dumpHeader = getDumpFileContentsRawTokensHeader(files);
+            fdump << getDumpFileContentsRawTokens(dumpHeader, dumpFooter);
+            mSuppressions.nomsg.dump(fdump);
+            // dumped all configs, close root </dumps> element now
             fdump << "</dumps>" << std::endl;
             fdump.close();
         }
 
+        if (!mSettings.plistOutput.empty()) {
+            mLogger->setPlistFilenames(std::move(files));
+        }
+
         executeAddons(dumpFile, file);
     } catch (const TerminateException &) {
         // Analysis is terminated
@@ -1892,16 +1927,39 @@ bool CppCheck::isPremiumCodingStandardId(const std::string& id) const {
     return false;
 }
 
-std::string CppCheck::getDumpFileContentsRawTokens(const std::vector<std::string>& files, const simplecpp::TokenList& tokens1) const {
+std::string CppCheck::getDumpFileContentsRawTokens(const std::vector<std::string>& files, const simplecpp::TokenList& tokens1) const
+{
+    std::string header = getDumpFileContentsRawTokensHeader(files);
+    std::string footer = getDumpFileContentsRawTokensFooter(tokens1);
+    return getDumpFileContentsRawTokens(header, footer);
+}
+
+std::string CppCheck::getDumpFileContentsRawTokens(const std::string& header, const std::string& footer)
+{
     std::string dumpProlog;
     dumpProlog += "  <rawtokens>\n";
+    dumpProlog += header;
+    dumpProlog += footer;
+    dumpProlog += "  </rawtokens>\n";
+    return dumpProlog;
+}
+
+std::string CppCheck::getDumpFileContentsRawTokensHeader(const std::vector<std::string>& files) const
+{
+    std::string dumpProlog;
     for (unsigned int i = 0; i < files.size(); ++i) {
         dumpProlog += "    <file index=\"";
         dumpProlog += std::to_string(i);
         dumpProlog += "\" name=\"";
         dumpProlog += ErrorLogger::toxml(Path::getRelativePath(files[i], mSettings.basePaths));
         dumpProlog += "\"/>\n";
     }
+    return dumpProlog;
+}
+
+std::string CppCheck::getDumpFileContentsRawTokensFooter(const simplecpp::TokenList& tokens1)
+{
+    std::string dumpProlog;
     for (const simplecpp::Token *tok = tokens1.cfront(); tok; tok = tok->next) {
         dumpProlog += "    <tok ";
 
@@ -1913,7 +1971,7 @@ std::string CppCheck::getDumpFileContentsRawTokens(const std::vector<std::string
         dumpProlog += std::to_string(tok->location.line);
         dumpProlog += "\" ";
 
-        dumpProlog +="column=\"";
+        dumpProlog += "column=\"";
         dumpProlog += std::to_string(tok->location.col);
         dumpProlog += "\" ";
 
@@ -1923,6 +1981,5 @@ std::string CppCheck::getDumpFileContentsRawTokens(const std::vector<std::string
 
         dumpProlog += "/>\n";
     }
-    dumpProlog += "  </rawtokens>\n";
     return dumpProlog;
 }

lib/cppcheck.h

@@ -153,6 +153,9 @@ class CPPCHECKLIB CppCheck {
      * @brief Get dumpfile <rawtokens> contents, this is only public for testing purposes
      */
     std::string getDumpFileContentsRawTokens(const std::vector<std::string>& files, const simplecpp::TokenList& tokens1) const;
+    static std::string getDumpFileContentsRawTokens(const std::string& header, const std::string& footer);
+    std::string getDumpFileContentsRawTokensHeader(const std::vector<std::string>& files) const;
+    static std::string getDumpFileContentsRawTokensFooter(const simplecpp::TokenList& tokens1);
 
     std::string getLibraryDumpData() const;
 
@@ -183,7 +186,7 @@ class CPPCHECKLIB CppCheck {
      */
     unsigned int checkFile(const FileWithDetails& file, const std::string &cfgname);
 
-    void checkPlistOutput(const FileWithDetails& file, const std::vector<std::string>& files);
+    void checkPlistOutput(const FileWithDetails& file);
 
     /**
      * @brief Check a file using buffer

lib/errorlogger.cpp

@@ -821,20 +821,15 @@ std::string ErrorLogger::toxml(const std::string &str)
     return xml;
 }
 
-std::string ErrorLogger::plistHeader(const std::string &version, const std::vector<std::string> &files)
+std::string ErrorLogger::plistHeader(const std::string &version)
 {
     std::ostringstream ostr;
     ostr << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n"
          << "<!DOCTYPE plist PUBLIC \"-//Apple Computer//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\r\n"
          << "<plist version=\"1.0\">\r\n"
          << "<dict>\r\n"
          << " <key>clang_version</key>\r\n"
-         << "<string>cppcheck version " << version << "</string>\r\n"
-         << " <key>files</key>\r\n"
-         << " <array>\r\n";
-    for (const std::string & file : files)
-        ostr << "  <string>" << ErrorLogger::toxml(file) << "</string>\r\n";
-    ostr << " </array>\r\n"
+         << " <string>cppcheck version " << version << "</string>\r\n"
          << " <key>diagnostics</key>\r\n"
          << " <array>\r\n";
     return ostr.str();

lib/errorlogger.h

@@ -28,6 +28,7 @@
 #include <ctime>
 #include <list>
 #include <set>
+#include <sstream>
 #include <string>
 #include <utility>
 #include <vector>
@@ -271,12 +272,19 @@ class CPPCHECKLIB ErrorLogger {
      */
     static std::string toxml(const std::string &str);
 
-    static std::string plistHeader(const std::string &version, const std::vector<std::string> &files);
+    static std::string plistHeader(const std::string &version);
     static std::string plistData(const ErrorMessage &msg);
-    static const char *plistFooter() {
-        return " </array>\r\n"
-               "</dict>\r\n"
-               "</plist>";
+    static std::string plistFooter(const std::vector<std::string>& files) {
+        std::ostringstream ostr;
+        ostr << " </array>\r\n"
+             << " <key>files</key>\r\n"
+             << " <array>\r\n";
+        for (const std::string& file : files)
+            ostr << "  <string>" << ErrorLogger::toxml(file) << "</string>\r\n";
+        ostr << " </array>\r\n"
+             << "</dict>\r\n"
+             << "</plist>";
+        return ostr.str();
     }
 
     static bool isCriticalErrorId(const std::string& id) {