/src/compiler/package.cpp
C++ | 474 lines | 390 code | 57 blank | 27 comment | 112 complexity | c6010854aa5d42089e8614531647b616 MD5 | raw file
- /*
- +----------------------------------------------------------------------+
- | HipHop for PHP |
- +----------------------------------------------------------------------+
- | Copyright (c) 2010 Facebook, Inc. (http://www.facebook.com) |
- +----------------------------------------------------------------------+
- | This source file is subject to version 3.01 of the PHP license, |
- | that is bundled with this package in the file LICENSE, and is |
- | available through the world-wide-web at the following url: |
- | http://www.php.net/license/3_01.txt |
- | If you did not receive a copy of the PHP license and are unable to |
- | obtain it through the world-wide-web, please send a note to |
- | license@php.net so we can mail you a copy immediately. |
- +----------------------------------------------------------------------+
- */
- #include <compiler/package.h>
- #include <sys/types.h>
- #include <sys/stat.h>
- #include <unistd.h>
- #include <dirent.h>
- #include <util/process.h>
- #include <util/util.h>
- #include <compiler/analysis/analysis_result.h>
- #include <compiler/parser/parser.h>
- #include <util/logger.h>
- #include <util/json.h>
- #include <compiler/analysis/symbol_table.h>
- #include <compiler/analysis/variable_table.h>
- #include <compiler/option.h>
- #include <util/db_conn.h>
- #include <util/db_query.h>
- #include <util/exception.h>
- #include <util/preprocess.h>
- using namespace HPHP;
- using namespace std;
- ///////////////////////////////////////////////////////////////////////////////
- // statics
- void (*Package::m_hookHandler)(Package *package, const char *path,
- HphpHookUniqueId id);
- ///////////////////////////////////////////////////////////////////////////////
- Package::Package(const char *root, bool bShortTags /* = true */,
- bool bAspTags /* = false */)
- : m_bShortTags(bShortTags), m_bAspTags(bAspTags), m_files(4000),
- m_lineCount(0), m_charCount(0) {
- m_root = root;
- if (!m_root.empty() && m_root[m_root.size() - 1] != '/') m_root += "/";
- m_ar = AnalysisResultPtr(new AnalysisResult());
- m_fileCache = FileCachePtr(new FileCache());
- }
- void Package::addAllFiles(bool force) {
- if (Option::PackageDirectories.empty() && Option::PackageFiles.empty()) {
- addDirectory("/", force);
- } else {
- for (set<string>::const_iterator iter = Option::PackageDirectories.begin();
- iter != Option::PackageDirectories.end(); ++iter) {
- addDirectory(*iter, force);
- }
- for (set<string>::const_iterator iter = Option::PackageFiles.begin();
- iter != Option::PackageFiles.end(); ++iter) {
- addSourceFile((*iter).c_str());
- }
- }
- }
- void Package::addSourceFile(const char *fileName) {
- ASSERT(fileName && *fileName);
- m_files.add(Util::canonicalize(fileName).c_str());
- }
- void Package::addListFiles(const char *listFileName) {
- ASSERT(listFileName && *listFileName);
- FILE *f = fopen(listFileName, "r");
- if (f == NULL) {
- throw Exception("Unable to open %s: %s", listFileName,
- Util::safe_strerror(errno).c_str());
- }
- char fileName[PATH_MAX];
- while (fgets(fileName, sizeof(fileName), f)) {
- int len = strlen(fileName);
- if (fileName[len - 1] == '\n') fileName[len-1] = '\0';
- addSourceFile(fileName);
- }
- fclose(f);
- }
- void Package::addStaticFile(const char *fileName) {
- ASSERT(fileName && *fileName);
- m_extraStaticFiles.insert(fileName);
- }
- void Package::addStaticDirectory(const std::string path) {
- m_staticDirectories.insert(path);
- }
- void Package::addDirectory(const std::string &path, bool force) {
- addDirectory(path.c_str(), force);
- }
- void Package::addDirectory(const char *path, bool force) {
- m_directories.insert(path);
- addDirectory(path, "*.php", force);
- #ifdef HAVE_PHPT
- addDirectory(path, "*.phpt", force);
- #endif
- addDirectory(path, "", force); // look for PHP files without postfix
- if (m_hookHandler) {
- m_hookHandler(this, path, onPackageAddDirectory);
- }
- }
- void Package::addDependencyParents(const char *path, const char *postfix,
- DependencyGraph::KindOf kindOf) {
- vector<string> files;
- findFiles(files, path, postfix);
- DependencyGraphPtr dep = m_ar->getDependencyGraph();
- int rootSize = m_root.size();
- for (unsigned int i = 0; i < files.size(); i++) {
- const string &file = files[i];
- ASSERT(file.substr(0, rootSize) == m_root);
- dep->addParent(kindOf, "", file.substr(rootSize), ConstructPtr());
- }
- }
- void Package::findFiles(std::vector<std::string> &out, const char *path,
- const char *postfix) {
- ASSERT(postfix && *postfix);
- if (!path) path = "";
- if (*path == '/') path++;
- string fullPath = m_root + path;
- const char *argv[] = {"", "-L", (char*)fullPath.c_str(),
- "-name", (char*)postfix, NULL};
- string files;
- Process::Exec("find", argv, NULL, files);
- Util::split('\n', files.c_str(), out, true);
- }
- void Package::findPHPFiles(std::vector<std::string> &out, const char *path) {
- if (!path) path = "";
- if (*path == '/') path++;
- string fullPath = m_root + path;
- const char *argv[] = {"", "-L", (char*)fullPath.c_str(),
- "-type", "f",
- "-regex", ".*/[A-Za-z0-9_\\-]+",
- "-not", "-regex", ".*/\\.svn/.*",
- "-not", "-regex", ".*/\\.git/.*",
- /* Do not use [A-Z] below. That seems to be
- broken outside of the C locale, and we dont
- know what locale find will run in */
- "-not", "-regex", ".*/[ABCDEFGHIJKLMNOPQRSTUVWXYZ]+",
- "-not", "-regex", ".*/tags",
- "-exec", "perl", "-n", "-e",
- "/php\\s*$/ && print $ARGV.\"\\n\"; exit",
- "{}", ";",
- NULL};
- string files;
- Process::Exec("find", argv, NULL, files);
- Util::split('\n', files.c_str(), out, true);
- }
- void Package::findNonPHPFiles(vector<string> &out, const char *path,
- bool exclude) {
- if (!path) path = "";
- if (*path == '/') path++;
- string fullPath = m_root + path;
- DIR *dir = opendir(fullPath.c_str());
- if (dir == NULL) {
- Logger::Error("findNonPHPFiles: unable to open directory %s",
- fullPath.c_str());
- return;
- }
- dirent *e;
- while (e = readdir(dir)) {
- char *ename = e->d_name;
- if (strcmp(ename, ".") == 0 || strcmp(ename, "..") == 0) {
- continue;
- }
- string fe = fullPath +
- (fullPath[fullPath.length() - 1] != '/' ? "/" : "") + ename;
- struct stat se;
- if (stat(fe.c_str(), &se) != 0) {
- // unable to stat, maybe a broken symbolic link
- // do not include it
- continue;
- }
- if ((se.st_mode & S_IFMT) == S_IFDIR) {
- if (strcmp(ename, ".svn") && strcmp(ename, ".git")) {
- string subdir = path;
- if (subdir[subdir.length() - 1] != '/') subdir += '/';
- subdir += ename;
- findNonPHPFiles(out, subdir.c_str(), exclude);
- }
- continue;
- }
- if (strcmp(ename, "tags") == 0) {
- continue;
- }
- size_t len = strlen(ename);
- if (len >= 4 && ename[len - 4] == '.' && ename[len - 3] == 'p' &&
- ename[len - 2] == 'h' && ename[len - 1] == 'p') {
- // ending with .php
- continue;
- }
- #ifdef HAVE_PHPT
- if (len >= 5 && ename[len - 5] == '.' && ename[len - 4] == 'p' &&
- ename[len - 3] == 'h' && ename[len - 2] == 'p' &&
- ename[len - 1] == 't') {
- // ending with .phpt
- continue;
- }
- #endif
- string fullname = string(path) + "/" + ename;
- if (!exclude ||
- Option::PackageExcludeStaticFiles.find(fullname) ==
- Option::PackageExcludeStaticFiles.end()) {
- out.push_back(fe);
- }
- }
- closedir(dir);
- }
- void Package::addDirectory(const char *path, const char *postfix, bool force) {
- ASSERT(path && *path);
- if (*path == '/') path++;
- vector<string> files;
- if (postfix && *postfix) {
- findFiles(files, path, postfix);
- } else {
- findPHPFiles(files, path);
- }
- for (unsigned int i = 0; i < files.size(); i++) {
- const string &file = files[i];
- bool excluded = false;
- if (!force) {
- for (set<string>::const_iterator iter =
- Option::PackageExcludeDirs.begin();
- iter != Option::PackageExcludeDirs.end(); ++iter) {
- if (file.find(*iter) == Option::RootDirectory.size()) {
- excluded = true;
- break;
- }
- }
- }
- if (!excluded) {
- ASSERT(file.substr(0, m_root.size()) == m_root);
- string name = file.substr(m_root.size());
- if (Option::PackageExcludeFiles.find(name) ==
- Option::PackageExcludeFiles.end()) {
- m_files.add(name.c_str());
- }
- }
- }
- }
- void Package::getFiles(std::vector<std::string> &files) const {
- files.clear();
- files.reserve(m_files.size());
- for (unsigned int i = 0; i < m_files.size(); i++) {
- const char *fileName = m_files.at(i);
- files.push_back(fileName);
- }
- }
- FileCachePtr Package::getFileCache() {
- for (set<string>::const_iterator iter = m_directories.begin();
- iter != m_directories.end(); ++iter) {
- vector<string> files;
- findNonPHPFiles(files, iter->c_str(), true);
- for (unsigned int i = 0; i < files.size(); i++) {
- string &file = files[i];
- string rpath = file.substr(m_root.size());
- if (!m_fileCache->fileExists(rpath.c_str())) {
- Logger::Verbose("saving %s", file.c_str());
- m_fileCache->write(rpath.c_str(), file.c_str());
- }
- }
- }
- for (set<string>::const_iterator iter = m_staticDirectories.begin();
- iter != m_staticDirectories.end(); ++iter) {
- vector<string> files;
- findNonPHPFiles(files, iter->c_str(), false);
- for (unsigned int i = 0; i < files.size(); i++) {
- string &file = files[i];
- string rpath = file.substr(m_root.size());
- if (!m_fileCache->fileExists(rpath.c_str())) {
- Logger::Verbose("saving %s", file.c_str());
- m_fileCache->write(rpath.c_str(), file.c_str());
- }
- }
- }
- for (set<string>::const_iterator iter = m_extraStaticFiles.begin();
- iter != m_extraStaticFiles.end(); ++iter) {
- const char *file = iter->c_str();
- if (!m_fileCache->fileExists(file)) {
- string fullpath = m_root + file;
- Logger::Verbose("saving %s", fullpath.c_str());
- m_fileCache->write(file, fullpath.c_str());
- }
- }
- return m_fileCache;
- }
- ///////////////////////////////////////////////////////////////////////////////
- bool Package::parse() {
- hphp_const_char_set files;
- for (unsigned int i = 0; i < m_files.size(); i++) {
- const char *fileName = m_files.at(i);
- if (files.find(fileName) == files.end()) {
- files.insert(fileName);
- if (!parseImpl(fileName)) return false;
- }
- }
- return true;
- }
- bool Package::parse(const char *fileName) {
- return parseImpl(m_files.add(fileName));
- }
- bool Package::parseImpl(const char *fileName) {
- ASSERT(fileName);
- if (fileName[0] == 0) return false;
- string fullPath;
- if (fileName[0] == '/') {
- fullPath = fileName;
- } else {
- fullPath = m_root + fileName;
- }
- struct stat sb;
- if (stat(fullPath.c_str(), &sb)) {
- Logger::Error("Unable to stat file %s", fullPath.c_str());
- return false;
- }
- try {
- ifstream f(fullPath.c_str());
- stringstream ss;
- istream *is = Option::EnableXHP ? preprocessXHP(f, ss, fullPath) : &f;
- Scanner scanner(new ylmm::basic_buffer(*is, false, true),
- m_bShortTags, m_bAspTags);
- Logger::Verbose("parsing %s ...", fullPath.c_str());
- ParserPtr parser(new Parser(scanner, fileName, sb.st_size, m_ar));
- if (parser->parse()) {
- throw Exception("Unable to parse file: %s\n%s", fullPath.c_str(),
- parser->getMessage().c_str());
- }
- m_lineCount += parser->line1();
- struct stat fst;
- stat(fullPath.c_str(), &fst);
- m_charCount += fst.st_size;
- } catch (std::runtime_error) {
- Logger::Error("Unable to open file %s", fullPath.c_str());
- return false;
- }
- if (!m_fileCache->fileExists(fileName) &&
- m_extraStaticFiles.find(fileName) == m_extraStaticFiles.end()) {
- if (Option::CachePHPFile) {
- m_fileCache->write(fileName, fullPath.c_str()); // name + content
- } else {
- m_fileCache->write(fileName); // just name, without content
- }
- }
- return true;
- }
- ///////////////////////////////////////////////////////////////////////////////
- void Package::saveStatsToFile(const char *filename, int totalSeconds) const {
- ofstream f(filename);
- if (f) {
- JSON::OutputStream o(f);
- JSON::MapStream ms(o);
- ms.add("FileCount", getFileCount())
- .add("LineCount", getLineCount())
- .add("CharCount", getCharCount())
- .add("FunctionCount", m_ar->getFunctionCount())
- .add("ClassCount", m_ar->getClassCount())
- .add("TotalTime", totalSeconds);
- if (getLineCount()) {
- ms.add("AvgCharPerLine", getCharCount() / getLineCount());
- }
- if (m_ar->getFunctionCount()) {
- ms.add("AvgLinePerFunc", getLineCount()/m_ar->getFunctionCount());
- }
- std::map<std::string, int> counts;
- SymbolTable::CountTypes(counts);
- m_ar->countReturnTypes(counts);
- ms.add("SymbolTypes");
- o << counts;
- ms.add("VariableTableFunctions");
- JSON::ListStream ls(o);
- BOOST_FOREACH(const std::string &f, m_ar->m_variableTableFunctions) {
- ls << f;
- }
- ls.done();
- ms.done();
- f.close();
- }
- }
- int Package::saveStatsToDB(ServerDataPtr server, int totalSeconds,
- const std::string &branch, int revision) const {
- std::map<std::string, int> counts;
- SymbolTable::CountTypes(counts);
- m_ar->countReturnTypes(counts);
- ostringstream sout;
- JSON::OutputStream o(sout);
- o << counts;
- DBConn conn;
- conn.open(server);
- const char *sql = "INSERT INTO hphp_run (branch, revision, file, line, "
- "byte, program, function, class, types, time)";
- DBQuery q(&conn, sql);
- q.insert("'%s', %d, %d, %d, %d, %d, %d, %d, '%s', %d",
- branch.c_str(), revision,
- getFileCount(), getLineCount(), getCharCount(),
- 1, m_ar->getFunctionCount(),
- m_ar->getClassCount(), sout.str().c_str(), totalSeconds);
- q.execute();
- return conn.getLastInsertId();
- }
- void Package::commitStats(ServerDataPtr server, int runId) const {
- DBConn conn;
- conn.open(server);
- {
- DBQuery q(&conn, "UPDATE hphp_dep");
- q.setField("parent_file = parent");
- q.filterBy("run = %d", runId);
- q.filterBy("kind IN ('PHPInclude', 'PHPTemplate')");
- q.execute();
- }
- {
- DBQuery q(&conn, "UPDATE hphp_run");
- q.setField("committed = 1");
- q.filterBy("id = %d", runId);
- q.execute();
- }
- }