/mvndep.py

Large files files are truncated, but you can click here to view the full file

#! /usr/bin/env python3

import json
import os, subprocess, argparse, re
from dateutil.parser import parse
import requests

# from lxml import html
FIX_PROPERTIES_UNUSED_VERSIONS = 'unused'

FIX_LIBS_LATEST_COMMON_VERSIONS = 'llibs'
FIX_LIBS_SET_VERSIONS = 'libs'
FIX_LIBS_FROM_REPO = 'rlibs'

FIX_PARENTS_SET_VERSIONS = 'parents'
FIX_PARENTS_FROM_REPO = 'rparents'

FIX_PLUGINS_SET_VERSION = 'plugins'
FIX_PLUGINS_FROM_REPO = 'rplugins'

ACTIONS = [FIX_PROPERTIES_UNUSED_VERSIONS,
           FIX_LIBS_LATEST_COMMON_VERSIONS,
           FIX_LIBS_SET_VERSIONS,
           FIX_LIBS_FROM_REPO,
           FIX_PARENTS_SET_VERSIONS,
           FIX_PARENTS_FROM_REPO,
           FIX_PLUGINS_SET_VERSION,
           FIX_PLUGINS_FROM_REPO]

CONFIG_FILENAME = '.mvndep.json'

POM_NAME = 'pom.xml'

SUBST_VER_RE = re.compile(r'{(?P<version>.+)}')
VER_RE = re.compile(r'<version>(?P<version>.+)</version>')
PROPS_START_RE = re.compile(r'<properties>')
PROPS_END_RE = re.compile(r'</properties>')
PROPS_VERSION_RE = re.compile(r'<version(?P<name>.+)>.+</version.+>')
PARENT_START_RE = re.compile(r'<parent>')
PARENT_END_RE = re.compile(r'</parent>')
GROUP_ID_RE = re.compile(r'<groupId>(?P<group>.+)</groupId>')
ARTIFACT_ID_RE = re.compile(r'<artifactId>(?P<artifact>.+)</artifactId>')
VERSION_RE = re.compile(r'<version>(?P<version>.+)</version>')
VERSION_DATE_TIME_RE = re.compile(r'<a href="(?P<version>.+)/">(?P<version2>.+)/</a>.+(?P<date_time>\d\d\d\d-\d\d-\d\d \d\d:\d\d).+')
VERSION_DATE_TIME_AF_RE = re.compile(r'<a href="(?P<version>.+)/">(?P<version2>.+)/</a>.+(?P<date_time>\d?\d-\S\S\S-\d\d\d\d \d\d:\d\d).+')
FOR_PROJECT_RE = re.compile(r'\[INFO].+@ (?P<project>[\S]+)')
MODULES_START_RE = re.compile(r'<modules>')


REPOS = ['https://repo1.maven.org/maven2/',
         'https://oss.sonatype.org/content/groups/public/'
        ]

REPO2 = ['https://repo1.maven.org/maven2/org/apache/maven/plugins/',
        'https://repo1.maven.org/maven2/org/codehaus/mojo/',
        'https://repo1.maven.org/maven2/org/codehaus/gmaven/',
        'https://central.maven.org/maven2/org/jvnet/jaxb2/maven2/',
        'https://central.maven.org/maven2/com/spotify/',
        'https://central.maven.org/maven2/org/apache/cxf/',
        'https://central.maven.org/maven2/org/springframework/boot/',
        'https://central.maven.org/maven2/org/apache/tomcat/maven/'
       ]


class Version:

    def __init__(self, str_version):
        self.vers = []
        self.rest = []
        self.version = str_version
        _vers = str_version.split('.')
        for _ver in _vers:
            if len(self.rest) > 0:
                self.rest.append(_ver)
                continue
            if _ver.isdigit():
                self.vers.append(int(_ver))
                continue
            start = ''
            end = ''
            for c in _ver:
                if len(end) == 0 and c.isdigit():
                    start = start + c
                else:
                    end = end + c
            if len(start) > 0:
                self.vers.append(int(start))
            if len(end) > 0:
                self.rest.append(end)

    def compare(self, version):
        'test, if this version is newer than version in parameter'
        # print(self.vers, self.rest, version.vers, version.rest)
        if self.version == version.version:
            return False
        mlen = min(len(self.vers), len(version.vers))
        for i in range(0, mlen):
            if self.vers[i] > version.vers[i]:
                return True
            if self.vers[i] < version.vers[i]:
                return False
        if len(self.vers) > len(version.vers):
            return True
        if len(self.vers) < len(version.vers):
            return False
        mlen = min(len(self.rest), len(version.rest))
        for i in range(0, mlen):
            if self.rest[i] > version.rest[i]:
                return True
            if self.rest[i] < version.rest[i]:
                return False
        if len(self.rest) > len(version.rest):
            return True
        if len(self.rest) < len(version.rest):
            return False
        return False

    def __hash__(self):
        return hash(self.version)

    def __eq__(self, other):
        return self.version == other.version

    def __str__(self):
        return self.version

    def __repr__(self):
        return str(self)


class Library:

    def __init__(self, str_library):
        items = str_library.split(':')
        if len(items) < 1:
            raise ValueError('%s should be groupId:artifactId:version, the required is at least groupId or artifactId' % (str))
        self.groupId = items[0]
        self.artifactId = None if len(items) <= 1 else items[1]
        self.version = None if len(items) <= 2 else items[2]
        self.library = str_library

    def __hash__(self):
        return hash(self.library)

    def __eq__(self, other):
        return self.library == other.library

    def __str__(self):
        return self.library

    def __repr__(self):
        return str(self)


class Libraries:

    def __init__(self, libs):
        self.map_group_artifact_version = {}
        self.map_artifact_version = {}
        self.map_group_artifact = {}
        self.set_group = set([])
        for lib in libs:
            self.__add_lib(lib)

    def __add_lib(self, str_library):
        lib = Library(str_library)
        if lib.groupId and lib.artifactId and lib.version:
            self.map_group_artifact_version[lib.groupId + '.' + lib.artifactId] = lib.version
        elif lib.artifactId and lib.version:
            self.map_artifact_version[lib.artifactId] = lib.version
        elif lib.groupId and lib.artifactId:
            self.map_group_artifact[lib.groupId] = lib.artifactId
        else:
            self.set_group.add(lib.groupId)

    def contains_and_version(self, str_library):
        if str_library in self.map_group_artifact_version:
            return True, self.map_group_artifact_version[str_library]
        if str_library in self.map_artifact_version:
            return True, self.map_artifact_version[str_library]
        items = str_library.split('.')
        artifactId = items[-1]
        items = items[0:-1]
        groupId = '.'.join(items)
        if groupId in self.set_group:
            return True, None
        if groupId in self.map_group_artifact:
            return True, None
        if groupId + '.' + artifactId in self.map_group_artifact_version:
            return True, self.map_group_artifact_version[groupId + '.' + artifactId]
        return False, None

    @staticmethod
    def skip(libs, str_library):
        if not libs:
            return False
        result = libs.contains_and_version(str_library)
        if result[0]:
            return True
        return False

    @staticmethod
    def process(libs, str_library):
        if not libs:
            return True
        result = libs.contains_and_version(str_library)
        if result[0]:
            return True
        return False

    @staticmethod
    def version(libs, str_library):
        if not libs:
            return None
        result = libs.contains_and_version(str_library)
        if result[0]:
            return result[1]
        return None

    def __str__(self):
        return 'g-a-v: ' + str(self.map_group_artifact_version) + '\na-v: ' + str(self.map_artifact_version) + '\ng-a: ' + str(self.map_group_artifact) + '\ng: ' + str(self.set_group)

    def __repr__(self):
        return str(self)


def read_pom(path, name=None):
    if name:
        fname = os.path.join(path, name, POM_NAME)
    else:
        fname = os.path.join(path, POM_NAME)
    print("<- %s" % fname)
    with open(fname, 'r') as file:
        return file.readlines()


def read_poms(cfg, path, depth=1, pom=None):
    map_project_pom = {}
    map_project_path = {}
    print('in ', path)
    if pom:
        lines = read_pom(path)
        group_id, artifact_id = find_main_in_pom(lines)
        map_project_pom[artifact_id] = lines
        map_project_path[artifact_id] = os.path.join(path)
    for entry in os.scandir(path):
        if not entry.name.startswith('.') and entry.is_dir():
            if POM_NAME in os.listdir(entry.path):
                if cfg.projects and not entry.name in cfg.projects:
                    continue
                if cfg.skip_projects and entry.name in cfg.skip_projects:
                    continue
                lines = read_pom(path, entry.name)
                group_id, artifact_id = find_main_in_pom(lines)
                map_project_pom[artifact_id] = lines
                map_project_path[artifact_id] = os.path.join(path, entry.name)
    if depth > 1:
        for project in list(map_project_pom.keys()):
            _map_project_pom, _map_project_path = read_poms(cfg, map_project_path[project], depth - 1)
            map_project_pom.update(_map_project_pom)
            map_project_path.update(_map_project_path)
    return map_project_pom, map_project_path


def write_pom(path, pom, skip_lines=None):
    fname = os.path.join(path, POM_NAME)
    print("-> %s" % fname)
    with open(fname, 'w') as file:
        for linenum, line in enumerate(pom):
            if skip_lines and linenum in skip_lines:
                continue
            file.write(line)


def write_poms(map_project_pom, map_project_path, skip_lines=None):
    for project in map_project_pom:
        if skip_lines:
            write_pom(map_project_path[project], map_project_pom[project], skip_lines[project])
        else:
            write_pom(map_project_path[project], map_project_pom[project])
    return map_project_pom


def maven_anc_line(cfg, line, map_anc_versions_main):
  if line.startswith("[INFO] Ancestor"):
      line_ok = line[22:]
      line_ok_arr = line_ok.split(" <-")
      if cfg.verbosity >= 2:
          print("-- %s" % line_ok_arr[0])
      cols = line_ok_arr[0].split(":")
      lib = cols[0] + '.' + cols[1]
      version = cols[2]
      map_anc_versions_main[lib] = [Version(version)]


def maven_dep_line(cfg, line, map_lib_versions_main, map_lib_versions):
    if line.startswith('[INFO] |') or line.startswith('[INFO] +') or line.startswith('[INFO] \\'):
        line_ok = line[7:]
        if line_ok.startswith('+-') or line_ok.startswith('\-'):
            if cfg.verbosity >= 2:
                print("== %s" % line_ok[3:])
            cols = line_ok[3:].split(":")
            scope = cols[4 if len(cols) == 5 else 5]
            if cfg.skip_scopes and scope in cfg.skip_scopes:
                return
            if cfg.scopes and scope not in cfg.scopes:
                return
            lib = cols[0] + '.' + cols[1]
            version = cols[3 if len(cols) == 5 else 4]
            map_lib_versions_main[lib] = [Version(version)]
        else:
            line_ok = line_ok[line_ok.find('-'):]
            if cfg.verbosity >= 2:
                print("-- %s" % line_ok[2:])
            cols = line_ok[2:].split(":")
            scope = cols[4 if len(cols) == 5 else 5]
            if cfg.skip_scopes and scope in cfg.skip_scopes:
                return
            if cfg.scopes and scope not in cfg.scopes:
                return
            lib = cols[0] + '.' + cols[1]
            version = cols[3 if len(cols) == 5 else 4]
            if lib in map_lib_versions:
                map_lib_versions[lib] = map_lib_versions[lib] + [Version(version)]
            else:
                map_lib_versions[lib] = [Version(version)]


def maven_dep_pom(cfg, path, pom=None):
    map2_project_lib_versions_main = {}
    map2_project_lib_versions = {}
    map_lib_versions_main = {}
    map_lib_versions = {}
    project = None
    if pom:
        pipe = subprocess.Popen(['mvn', '-f', pom, 'dependency:tree'], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    else:
        pipe = subprocess.Popen(['mvn', 'dependency:tree'], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    text = pipe.communicate()[0].decode("utf-8")
#     print(text)
    for line in text.split("\n"):
        if line.find('[ERROR]') >= 0:
            print(line)
            continue
        result = FOR_PROJECT_RE.search(line)
        if result:
            print(line)
            if project:
                map2_project_lib_versions_main[project] = map_lib_versions_main
                map2_project_lib_versions[project] = map_lib_versions
            map_lib_versions_main = {}
            map_lib_versions = {}
            project = result.group('project')
            if cfg.projects and not project in cfg.projects:
                project = None
            if cfg.skip_projects and project in cfg.skip_projects:
                project = None
            print("project %s" % (project))
        else:
            maven_dep_line(cfg, line, map_lib_versions_main, map_lib_versions)
    if project:
        map2_project_lib_versions_main[project] = map_lib_versions_main
        map2_project_lib_versions[project] = map_lib_versions
    return (map2_project_lib_versions_main, map2_project_lib_versions)

  
def maven_anc_pom(cfg, path, pom=None):
    map2_project_lib_versions_main = {}
    map_lib_versions_main = {}
    project = None
    if pom:
        pipe = subprocess.Popen(['mvn', '-f', pom, 'dependency:display-ancestors'], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    else:
        pipe = subprocess.Popen(['mvn', 'dependency:display-ancestors'], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    text = pipe.communicate()[0].decode("utf-8")
#     print(text)
    for line in text.split("\n"):
        if line.find('[ERROR]') >= 0:
            print(line)
            continue
        result = FOR_PROJECT_RE.search(line)
        if result:
            print(line)
            if project:
                map2_project_lib_versions_main[project] = map_lib_versions_main
            map_lib_versions_main = {}
            project = result.group('project')
            if cfg.projects and not project in cfg.projects:
                project = None
            if cfg.skip_projects and project in cfg.skip_projects:
                project = None
            print("project %s" % (project))
        else:
            maven_anc_line(cfg, line, map_lib_versions_main)
    if project:
        map2_project_lib_versions_main[project] = map_lib_versions_main
    return (map2_project_lib_versions_main)

  
def maven_anc_tree(cfg, path):
    map2_project_lib_versions_main = {}
    for entry in os.scandir(path):
        if not entry.name.startswith('.') and entry.is_dir():
            if POM_NAME in os.listdir(entry.path):
                if cfg.projects and not entry.name in cfg.projects:
                    continue
                if cfg.skip_projects and entry.name in cfg.skip_projects:
                    continue
                print("dir %s" % (entry.path))
                _map2_project_lib_versions_main = maven_anc_pom(cfg, entry.path, None)
                if cfg.verbosity >= 2:
                    print(_map2_project_lib_versions_main)
                map2_project_lib_versions_main.update(_map2_project_lib_versions_main)
    return (map2_project_lib_versions_main)


def maven_dep_tree(cfg, path):
    map2_project_lib_versions_main = {}
    map2_project_lib_versions = {}
    for entry in os.scandir(path):
        if not entry.name.startswith('.') and entry.is_dir():
            if POM_NAME in os.listdir(entry.path):
                if cfg.projects and not entry.name in cfg.projects:
                    continue
                if cfg.skip_projects and entry.name in cfg.skip_projects:
                    continue
                print("dir %s" % (entry.path))
                _map2_project_lib_versions_main, _map2_project_lib_versions = maven_dep_pom(cfg, entry.path, None)
                if cfg.verbosity >= 2:
                    print(_map2_project_lib_versions_main)
                    print(_map2_project_lib_versions)
                map2_project_lib_versions_main.update(_map2_project_lib_versions_main)
                map2_project_lib_versions.update(_map2_project_lib_versions)
    return (map2_project_lib_versions_main, map2_project_lib_versions)


def maven_ancestors(cfg):
  if cfg.pom:
    return maven_anc_pom(cfg, cfg.dir, cfg.pom)
  else:
    return maven_anc_tree(cfg, cfg.dir)


def maven_dependencies(cfg):
    if cfg.pom:
        return maven_dep_pom(cfg, cfg.dir, cfg.pom)
    else:
        return maven_dep_tree(cfg, cfg.dir)


def maven_plug_line(cfg, line, map_lib_versions_main):
    if line.startswith('[INFO] Plugin Resolved'):
        line_ok = line[24:]
        if cfg.verbosity >= 2:
            print("== %s" % line_ok)
        cols = line_ok.split("-")
        version = cols[-1][:-4]
        del cols[-1]
#         lib = 'org.apache.maven.plugins.' + '-'.join(cols)
        lib = '-'.join(cols)
        map_lib_versions_main[lib] = [Version(version)]


def maven_plug_pom(cfg, path, pom=None):
    map2_project_lib_versions_main = {}
    map_lib_versions_main = {}
    project = None
    if pom:
        if cfg.exclude_artifacts:
            pipe = subprocess.Popen(['mvn', '-f', pom, 'dependency:resolve-plugins', '-DexcludeArtifactIds=' + ','.join(cfg.exclude_artifacts)], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        else:
            pipe = subprocess.Popen(['mvn', '-f', pom, 'dependency:resolve-plugins'], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    else:
        if cfg.exclude_artifacts:
            pipe = subprocess.Popen(['mvn', 'dependency:resolve-plugins', '-DexcludeArtifactIds=' + ','.join(cfg.exclude_artifacts)], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        else:
            pipe = subprocess.Popen(['mvn', 'dependency:resolve-plugins'], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    text = pipe.communicate()[0].decode("utf-8")
#     print(text)
    for line in text.split("\n"):
        if line.find('[ERROR]') >= 0:
            print(line)
            continue
        result = FOR_PROJECT_RE.search(line)
        if result:
            if project:
                map2_project_lib_versions_main[project] = map_lib_versions_main
            map_lib_versions_main = {}
            project = result.group('project')
            if cfg.projects and not project in cfg.projects:
                project = None
            if cfg.skip_projects and project in cfg.skip_projects:
                project = None
            print("project %s" % (project))
        else:
            maven_plug_line(cfg, line, map_lib_versions_main)
    if project:
        map2_project_lib_versions_main[project] = map_lib_versions_main
    return map2_project_lib_versions_main


def maven_plug_tree(cfg, path):
    map2_project_lib_versions_main = {}
    for entry in os.scandir(path):
        if not entry.name.startswith('.') and entry.is_dir():
            if POM_NAME in os.listdir(entry.path):
                if cfg.projects and not entry.name in cfg.projects:
                    continue
                if cfg.skip_projects and entry.name in cfg.skip_projects:
                    continue
                print("dir %s" % (entry.path))
                _map2_project_lib_versions_main = maven_plug_pom(cfg, entry.path, None)
                if cfg.verbosity >= 2:
                    print(_map2_project_lib_versions_main)
                map2_project_lib_versions_main.update(_map2_project_lib_versions_main)
    return map2_project_lib_versions_main


def maven_plugins(cfg):
    if cfg.pom:
        return maven_plug_pom(cfg, cfg.dir, cfg.pom)
    else:
        return maven_plug_tree(cfg, cfg.dir)


def merge_libs(cfg, map2_project_lib_versions, map2_lib_version_projects=None):
    if not map2_lib_version_projects:
        map2_lib_version_projects_new = {}
    else:
        map2_lib_version_projects_new = map2_lib_version_projects.copy()
    for project in map2_project_lib_versions.keys():
        for lib, vers in map2_project_lib_versions[project].items():
            for ver in vers:
                if cfg.verbosity >= 2:
                    print("%s - %s - %s" % (project, lib, ver))
                if lib in map2_lib_version_projects_new:
                    ver_projects = map2_lib_version_projects_new[lib]
                    if ver in ver_projects:
                        ver_projects[ver] = ver_projects[ver] + [project]
                    else:
                        ver_projects[ver] = [project]
                else:
                    map2_lib_version_projects_new[lib] = {ver : [project]}
    return map2_lib_version_projects_new


def dump_libs_usage(text, map2_lib_version_projects_main, _all=True):
    print("-> %s " % text)
    libs = list(map2_lib_version_projects_main.keys())
    libs.sort()
    for lib in libs:
        map_version_projects = map2_lib_version_projects_main[lib]
        if _all:
            if len(map_version_projects) > 1:
                print("%s = %s  ****" % (lib, map_version_projects))
            else:
                print("%s = %s" % (lib, map_version_projects))
        else:
            if (len(map_version_projects) > 1):
                print("%s = %s" % (lib, map_version_projects))


def find_value_in_pom(pom, value):
    str_re = re.compile('<' + value + '>(?P<value>.+)</' + value + '>')
    for linenum, line in enumerate(pom):
        result = str_re.search(line)
        if result:
            return linenum, result.group('value')
    return None, None


def find_artifact_in_pom(pom, artifact, from_linenum=None):
    artifact_re = re.compile('<artifactId>' + artifact + '</artifactId>')
    found_line, prev_line, next_line = None, None, None
    found_linenum = None
    for linenum, line in enumerate(pom):
        if from_linenum:
            if linenum <= from_linenum + 1:
                continue
        if found_line:
            next_line = line
            break
        if artifact_re.search(line):
            found_line = line
            found_linenum = linenum
        else:
            prev_line = line
    return found_linenum, found_line, prev_line, next_line


def find_version_in_pom(pom, found_linenum, found_line, prev_line, next_line):
    result = SUBST_VER_RE.search(next_line)
    if result:
        version = result.group('version')
        return find_value_in_pom(pom, version)
    result = SUBST_VER_RE.search(prev_line)
    if result:
        version = result.group('version')
        return find_value_in_pom(pom, version)
    result = VER_RE.search(next_line)
    if result:
        return found_linenum + 1, result.group('version')
    result = VER_RE.search(prev_line)
    if result:
        return found_linenum - 1, result.group('version')
    return None, None


def find_versions_in_pom(pom):
    found_props = False
    map_verprop_linenum = {}
    for linenum, line in enumerate(pom):
        if found_props:
            if PROPS_END_RE.search(line):
                break
            result = PROPS_VERSION_RE.search(line)
            if result:
                map_verprop_linenum['version' + result.group('name')] = linenum
        if PROPS_START_RE.search(line):
            found_props = True
    return map_verprop_linenum


def unused_versions_in_pom(pom, map_verprop_linenum):
    unused_pom_versions = dict(map_verprop_linenum)
    pom_versions_unused = map_verprop_linenum
    for verprop in pom_versions_unused.keys():
        verprop_re = re.compile('{' + verprop + '}')
        for line in pom:
            if verprop_re.search(line):
                unused_pom_versions.pop(verprop)
                break
    return unused_pom_versions


def find_main_in_pom(pom):
    group_id, artifact_id = None, None
    for line in pom:
        if group_id and artifact_id:
            break
        if MODULES_START_RE.search(line):
            break
        if PROPS_START_RE.search(line):
            break
        result = GROUP_ID_RE.search(line)
        if result:
            group_id = result.group('group')
            continue
        result = ARTIFACT_ID_RE.search(line)
        if result:
            artifact_id = result.group('artifact')
            continue
    return group_id, artifact_id


def find_parent_in_pom(pom):
    found_parent = False
    group_id, artifact_id, version, version_line = None, None, None, None
    for index, line in enumerate(pom):
        if found_parent:
            if PARENT_END_RE.search(line):
                break
            result = GROUP_ID_RE.search(line)
            if result:
                group_id = result.group('group')
                continue
            result = ARTIFACT_ID_RE.search(line)
            if result:
                artifact_id = result.group('artifact')
                continue
            result = VERSION_RE.search(line)
            if result:
                version = result.group('version')
                version_line = index
                continue
        if PARENT_START_RE.search(line):
            found_parent = True
    return group_id, artifact_id, version, version_line


def find_versions_maven_central(cfg, lib):
    list_ver_datetime = []
    if lib.startswith('cz.'):
        return list_ver_datetime
#     print('1111', lib)
    for repo in cfg.remote_urls:
        url = repo + lib.replace('.', '/') + '/'
#         print('2222', url)
        try:
            req = requests.get(url)
        except Exception as ex:
            print('For', url, 'there is error:', ex)
            return list_ver_datetime
#         print('2222b', req.status_code)
        if req.status_code == 200:
            break
    if req.status_code != 200:
        for repo in cfg.remote_urls:
            url = repo + lib.replace('.', '/') + '/'
            ix = url[:-1].rfind('/')
            if ix < 0:
                continue
            url = url[:ix] + '.' + url[ix + 1:]
#             print('3333', url)
            req = requests.get(url)
#             print('3333b', req.status_code)
            if req.status_code == 200:
                break
    if req.status_code != 200:
        print("Not found ", lib)
        return list_ver_datetime
#     print(req.content)
    lines = str(req.content).split('\\n')
#     tree = html.fromstring(req.content)
#     hrefs = tree.xpath('/html/body//a/@href')
    for line in lines:
        result = VERSION_DATE_TIME_RE.search(line)
        if result:
            version = result.group('version2')
            date_time = result.group('date_time')
            if version.startswith('.') or version.startswith('maven'):
                continue
            if version.endswith('/'):
                version = version[0:-1]
            list_ver_datetime.append((version, date_time))
        else:
            result = VERSION_DATE_TIME_AF_RE.search(line)
            if result:
                version = result.group('version2')
                date_time = result.group('date_time')
                if version.startswith('.') or version.startswith('maven'):
                    continue
                if version.endswith('/'):
                    version = version[0:-1]
                dt = parse(date_time)
                list_ver_datetime.append((version, str(dt)))
    return list_ver_datetime


central_maven_incorrect_datetimes = set(['commons-lang.commons-lang'])


def compare_based_on_version(lib):
    return True


def is_valid_lib(lib):
    if lib.startswith('200'):
        return False
    if lib.lower().find('alpha') >= 0 or lib.lower().find('beta') >= 0 or lib.lower().find('incubat') >= 0 or lib.lower().find('mistake') >= 0 or lib.find('-M') >= 0:
        return False
    return True


def find_latest_maven_central(cfg, lib):
    list_ver_datetime = find_versions_maven_central(cfg, lib)
    if not list_ver_datetime:
        return None
    if compare_based_on_version(lib):
        latest_version, latest_datetime = None, None
        for ver_datetime in list_ver_datetime:
            if not latest_version:
                if is_valid_lib(ver_datetime[0]):
                    latest_version = Version(ver_datetime[0])
                    latest_datetime = ver_datetime[1]
            elif is_valid_lib(ver_datetime[0]):
                version = Version(ver_datetime[0])
                if version.compare(latest_version):
                    latest_version = version
                    latest_datetime = ver_datetime[1]
        if latest_version:
            print('find_latest_maven_central lib=%s, ver=%s from %s' % (lib, latest_version, latest_datetime))
            return latest_version.version
        else:
            print('not find_latest_maven_central lib=%s' % (lib))
            return None
    else:
        latest_version, latest_datetime = None, None
        for ver_datetime in list_ver_datetime:
            if not latest_datetime or ver_datetime[1] > latest_datetime:
                latest_version = ver_datetime[0]
                latest_datetime = ver_datetime[1]
        print('find_latest_maven_central lib=%s, ver=%s from %s' % (lib, latest_version, latest_datetime))
        return latest_version


def find_pom_for_artifact_version(lib, project, map_project_pom, map_project_parents):
    pom = map_project_pom[project]
    _lib = lib.split('.')[-1]
    found_linenum, found_line, prev_line, next_line = find_artifact_in_pom(pom, _lib)
    if found_line:
#         print('found_line ', found_line, end = '')
#         print('prev_line ', prev_line, end = '')
#         print('next_line ', next_line, end = '')
        version_linenum, version = find_version_in_pom(pom, found_linenum, found_line, prev_line, next_line)
        if not version:
            found_linenum, found_line, prev_line, next_line = find_artifact_in_pom(pom, _lib, found_linenum)
            if found_line:
                version_linenum, version = find_version_in_pom(pom, found_linenum, found_line, prev_line, next_line)
        if version:
            return project, pom, version_linenum, version
    if not project in map_project_parents:
        return None, None, None, None
    for parent in map_project_parents[project]:
        pom = map_project_pom[parent]
        found_linenum, found_line, prev_line, next_line = find_artifact_in_pom(pom, _lib)
        if found_line:
#             print('found_line2 ', found_line, end = '')
#             print('prev_line2 ', prev_line, end = '')
#             print('next_line2 ', next_line, end = '')
            version_linenum, version = find_version_in_pom(pom, found_linenum, found_line, prev_line, next_line)
            if not version:
                found_linenum, found_line, prev_line, next_line = find_artifact_in_pom(pom, _lib, found_linenum)
                if found_line:
                    version_linenum, version = find_version_in_pom(pom, found_linenum, found_line, prev_line, next_line)
            if version:
                return parent, pom, version_linenum, version
    return None, None, None, None


def fix_latest_versions_in_poms(cfg, map_project_pom, map_project_parents, map2_lib_version_projects):
    map_project_pom_fixed = {}
    libs = list(map2_lib_version_projects.keys())
    libs.sort()
    for lib in libs:
        if len(map2_lib_version_projects[lib]) <= 1:
            continue
        if Libraries.skip(cfg.skip_libs, lib):
            continue
        if not Libraries.process(cfg.libs, lib):
            continue
        # print("%s = %s" % (lib, map2_lib_version_projects[lib]))
        latest_version = None
        for version, projects in map2_lib_version_projects[lib].items():
            if not latest_version or version.compare(latest_version):
                latest_version = version
        for version, projects in map2_lib_version_projects[lib].items():
            if latest_version == version:
                continue
            print("FIX for %s version %s -> %s in %s" % (lib, version, latest_version, projects))
            for project in projects:
                if not project in map_project_pom:
                    print('??? Missing ' + project + ' in ', map_project_pom.keys())
                    continue
                project_version, pom_version, version_linenum, version = find_pom_for_artifact_version(lib, project, map_project_pom, map_project_parents)
                if not version:
                    print("Not found artifactId (or version) %s in %s nor in it's parents" % (lib.split('.')[-1], project))
                    continue
                # print("Found version for artifactId %s in %s" % (lib.split('.')[-1], project_version))
                if version == latest_version.version:
                    print("FIXED line %d: %s -> %s in %s" % (version_linenum, version, latest_version, project_version))
                else:
                    print("TOFIX line %d: %s -> %s in %s" % (version_linenum, version, latest_version.version, project_version))
                    print(pom_version[version_linenum], end="")
                    if cfg.fix >= 2:
                        pom_version[version_linenum] = pom_version[version_linenum].replace(version, latest_version.version)
                        map_project_pom_fixed[project_version] = pom_version
                        print(pom_version[version_linenum], end="")
    return map_project_pom_fixed


def update_libraries_in_poms(cfg, map_project_pom, map_project_parents, map2_lib_version_projects, central_libs):
    map_project_pom_fixed = {}
    libs = list(map2_lib_version_projects.keys())
    libs.sort()
    for lib in libs:
        if Libraries.skip(cfg.skip_libs, lib):
            continue
        if central_libs:
            if not Libraries.process(cfg.libs, lib):
                continue
            if not lib in central_libs:
                continue
            new_version = central_libs[lib]
        else:
            new_version = Libraries.version(cfg.libs, lib)
            if not new_version:
                continue
#         print("%s = %s" % (lib, map2_lib_version_projects[lib]))
        for version, projects in map2_lib_version_projects[lib].items():
            if new_version == version.version:
                continue
            print("FIX for %s version %s -> %s in %s" % (lib, version, new_version, projects))
            for project in projects:
                if not project in map_project_pom:
                    print('??? Missing ' + project + ' in ', map_project_pom.keys())
                    continue
                project_version, pom_version, version_linenum, version = find_pom_for_artifact_version(lib, project, map_project_pom, map_project_parents)
                if not version:
                    print("Not found artifactId (or version) %s in %s nor in it's parents" % (lib.split('.')[-1], project))
                    continue
                # print("Found version for artifactId %s in %s" % (lib.split('.')[-1], project_version))
                if version == new_version:
                    print("FIXED line %d: %s -> %s in %s" % (version_linenum, version, new_version, project_version))
                else:
                    print("TOFIX line %d: %s -> %s in %s" % (version_linenum, version, new_version, project_version))
                    print(pom_version[version_linenum], end="")
                    if cfg.fix >= 2:
                        pom_version[version_linenum] = pom_version[version_linenum].replace(version, new_version)
                        map_project_pom_fixed[project_version] = pom_version
                        print(pom_version[version_linenum], end="")
    return map_project_pom_fixed


def update_parent_in_poms(cfg, map_project_pom, central_ancs=None):
    map_project_pom_fixed = {}
    for project, pom in map_project_pom.items():
        group_id, artifact_id, version, version_line = find_parent_in_pom(pom)
        if group_id and artifact_id and version and version_line:
            lib = group_id + "." + artifact_id
            if central_ancs:
                if lib in central_ancs:
                    new_version = central_ancs[lib]
                else:
                    new_version = None
            else:    
                new_version = Libraries.version(cfg.libs, lib)
            if new_version:
                print("TOFIX line %d: %s -> %s in %s" % (version_line, version, new_version, project))
                print(pom[version_line], end="")
                if cfg.fix >= 2:
                    pom[version_line] = pom[version_line].replace(version, new_version)
                    map_project_pom_fixed[project] = pom
                    print(pom[version_line], end="")
    return map_project_pom_fixed


def build_pom_hierarchy(map_project_path):
    map_project_parents = {}
    main_project = None
    for project, path in map_project_path.items():
        if path == '.':
            main_project = project
    for project, path in map_project_path.items():
#         print(project, ': ', path)
        path_list = path.split(os.sep)
        path_list.reverse()
#         print(path_list)
        parents = []
        project_first = True
        for _project in path_list:
            if project_first:
                project_first = False
                if _project != '.':
                    continue
            if _project == '.':
                if main_project:
                    parents.append(main_project)
            else:
                parents.append(_project)
        if parents:
            map_project_parents[project] = parents
#     print(map_project_parents)
    return map_project_parents


class Config:

    def __init__(self):
        self.default_skip_projects = []
        self.default_skip_libs = []
        self.default_skip_scopes = []
        self.default_pom = None
        self.default_depth = 1
        self.default_exclude_artifacts = []
        self.default_remote_urls = []
        
        if os.path.isfile(CONFIG_FILENAME):
            with open(CONFIG_FILENAME, encoding='utf-8') as config_file:
                default_config = json.loads(config_file.read())
                if 'skip_projects' in default_config:
                    self.default_skip_projects = default_config['skip_projects']
                    print("default skip projects", self.default_skip_projects)
                if 'skip_libs' in default_config:
                    self.default_skip_libs = default_config['skip_libs']
                    print("default skip libs", self.default_skip_libs)
                if 'skip_scopes' in default_config:
                    self.default_skip_scopes = default_config['skip_scopes']
                    print("default skip scopes", self.default_skip_scopes)
                if 'pom' in default_config:
                    self.default_pom = default_config['pom']
                    print("default pom", self.default_pom)
                if 'depth' in default_config:
                    self.default_depth = default_config['depth']
                    print("default depth", self.default_depth)
                if 'exclude_artifacts' in default_config:
                    self.default_exclude_artifacts = default_config['exclude_artifacts']
                    print("default exclude artifacts", self.default_exclude_artifacts)
                if 'remote_urls' in default_config:
                    self.default_remote_urls = default_config['remote_urls']
                    print("default remote urls", self.default_remote_urls)

        parser = argparse.ArgumentParser(description='Analyze Maven projects.')
        parser.add_argument('-d', '--dir', metavar='dir', default='.',
                            help='directory to scan for the Maven projects (default: the working directory)')
        parser.add_argument('-p', '--projects', metavar='project', nargs='+',
                            help='the Maven projects to analyze (default: all projects in selected directory)')
        parser.add_argument('-P', '--skip_projects', metavar='project', nargs='+', default=self.default_skip_projects,
                            help='the Maven projects to skip')
        parser.add_argument('-v', '--verbosity', action='count', default=0,
                            help='the verbosity of debug output')
        parser.add_argument('-f', '--fix', action='count', default=0,
                            help='fix the Maven projects\' to the latest libraries\' version')
        parser.add_argument('-t', '--type', default=FIX_LIBS_LATEST_COMMON_VERSIONS, choices=ACTIONS,
                            help='fix the Maven projects\' to the latest libraries\' version')
        parser.add_argument('-s', '--scopes', metavar='scope', nargs='+',
                            help='the Maven scopes to work with')
        parser.add_argument('-S', '--skip_scopes', metavar='scope', nargs='+', default=self.default_skip_scopes,
                            help='the Maven scopes to skip')
        parser.add_argument('-l', '--libs', dest='_libs', metavar='library', nargs='+',
                            help='the list of libraries to process (in the form groupId:artifactId:version)')
        parser.add_argument('-L', '--skip_libs', dest='_skip_libs', metavar='library', nargs='+', default=self.default_skip_libs,
                            help='the list of libraries to skip (in the form groupId:artifactId:version)')
        parser.add_argument('--pom', metavar='pom', default=self.default_pom,
                            help='the parent (main) pom.xml used instead of directory scanning')
        parser.add_argument('--depth', type=int, default=self.default_depth,
                            help='the depth of directories to scan for pom.xml')
        parser.add_argument('-X', '--exclude_artifacts', dest='exclude_artifacts', metavar='artifact', nargs='+', default=self.default_exclude_artifacts,
                            help='the list of artifacts to exclude (right now only plugins)')
        parser.add_argument('-r', '--remote', dest='_remote_urls', metavar='url', nargs='+', default=self.default_remote_urls,
                            help='consult the remote (central) maven repositories')
        parser.parse_args(namespace=self)

        self.libs = None if not self._libs else Libraries(self._libs)
        self.skip_libs = None if not self._skip_libs else Libraries(self._skip_libs)

        print("selected directory %s" % self.dir)
        print("selected projects %s" % self.projects)
        print("skip projects %s" % self.skip_projects)
        print("selected scopes %s" % self.scopes)
        print("skip scopes %s" % self.skip_scopes)
        print("libraries %s" % self.libs)
        print("skip libraries %s" % self.skip_libs)
        print("verbosity %d" % self.verbosity)
        print("fix %d" % self.fix)
        print("type %s" % self.type)
        print("pom %s" % self.pom)
        print("depth %d" % self.depth)
        print("exclude_artifacts %s" % self.exclude_artifacts)
        print("remote urls %s" % self._remote_urls)
        if self.type == FIX_PLUGINS_FROM_REPO:
            self.remote_urls = REPO2 + self._remote_urls
        else:
            self.remote_urls = REPOS + self._remote_urls


def main():
    cfg = Config()

    if cfg.type == FIX_LIBS_LATEST_COMMON_VERSIONS:
        map2_project_lib_versions_main, map2_project_lib_versions = maven_dependencies(cfg)
        if cfg.verbosity >= 1:
            print(map2_project_lib_versions_main)
            print(map2_project_lib_versions)
        map2_lib_version_projects_main = merge_libs(cfg, map2_project_lib_versions_main)
        if cfg.fix == 0:
            print("-------------------------------------------------------")
            dump_libs_usage("Main dependencies", map2_lib_version_projects_main)
        map2_lib_version_projects = merge_libs(cfg, map2_project_lib_versions, map2_lib_version_projects_main)
        if cfg.fix == 0:
            print("-------------------------------------------------------")
            dump_libs_usage("Conflict dependencies", map2_lib_version_projects, False)
            print("-------------------------------------------------------")

        if cfg.fix >= 1:
            map_project_pom, map_project_path = read_poms(cfg, cfg.dir, cfg.depth, cfg.pom)
            if cfg.verbosity >= 3:
                print(map_project_pom)
                print(map_project_path)
            map_project_parents = build_pom_hierarchy(map_project_path)
            map_project_pom_fixed = fix_latest_versions_in_poms(cfg, map_project_pom, map_project_parents, map2_lib_version_projects)
            if cfg.fix >= 2:
                write_poms(map_project_pom_fixed, map_project_path)

    elif cfg.type == FIX_LIBS_SET_VERSIONS and cfg.libs:
        map2_project_lib_versions_main, map2_project_lib_versions = maven_dependencies(cfg)
        if cfg.verbosity >= 1:
            print(map2_project_lib_versions_main)
        map2_lib_version_projects_main = merge_libs(cfg, map2_project_lib_versions_main)
        if cfg.verbosity >= 3:
            print(map2_lib_version_projects_main)

        map_project_pom, map_project_path = read_poms(cfg, cfg.dir, cfg.depth, cfg.pom)
        if cfg.verbosity >= 3:
            print(map_project_pom)
            print(map_project_path)
        map_project_parents = build_pom_hierarchy(map_project_path)
        map_project_pom_fixed = update_libraries_in_poms(cfg, map_project_pom, map_project_parents, map2_lib_version_projects_main, None)
        if cfg.fix >= 2:
            write_poms(map_project_pom_fixed, map_project_path)

    elif cfg.type == FIX_PROPERTIES_UNUSED_VERSIONS:
        map_project_pom, map_project_path = read_poms(cfg, cfg.dir, cfg.depth, cfg.pom)
        if cfg.verbosity >= 3:
            print(map_project_pom)
            print(map_project_path)
        map_project_parents = build_pom_hierarchy(map_project_path)
        map_project_linenums_unused = {}
        for project, pom in map_project_pom.items():
            map_verprop_linenum = find_versions_in_pom(pom)
            map_project_linenums_unused[project] = list(unused_versions_in_pom(pom, map_verprop_linenum).values())
        if map_project_linenums_unused and cfg.fix >= 2:
            write_poms(map_project_pom, map_project_path, map_project_linenums_unused)
        else:
            print(map_project_linenums_unused)

    elif cfg.type == FIX_PARENTS_SET_VERSIONS and cfg.libs:
        map_project_pom, map_project_path = read_poms(cfg, cfg.dir, cfg.depth, cfg.pom)
        if cfg.verbosity >= 3:
            print(map_project_pom)
            print(map_project_path)
        map_project_parents = build_pom_hierarchy(map_project_path)
        map_project_pom_fixed = update_parent_in_poms(cfg, map_project_pom)
        if cfg.fix >= 2:
            write_poms(map_project_pom_fixed, map_project_path)

    elif cfg.type == FIX_LIBS_FROM_REPO:
        map2_project_lib_versions_main, map2_project_lib_versions = maven_dependencies(cfg)
        if cfg.verbosity >= 1:
            print("---map2_project_lib_versions_main-----------------------------------------------------------------------------------------------------------")
            print(map2_project_lib_versions_main)
            print("---map2_project_lib_versions-----------------------------------------------------------------------------------------------------------")
            print(map2_project_lib_versions)

        map2_lib_version_projects_main = merge_libs(cfg, map2_project_lib_versions_main)
        if cfg.verbosity >= 3:
            print("---map2_lib_version_projects_main-----------------------------------------------------------------------------------------------------------")
            print(map2_lib_version_projects_main)
        central_libs = {}
        for lib in map2_lib_version_projects_main.keys():
            latest_version = find_latest_maven_central(cfg, lib)
            if latest_version:
                central_libs[lib] = latest_version
        if cfg.verbosity >= 2:
            print("---central_libs-----------------------------------------------------------------------------------------------------------")
            print(central_libs)

        if cfg.fix >= 1:
            map_project_pom, map_project_path = read_poms(cfg, cfg.dir, cfg.depth, cfg.pom)
            if cfg.verbosity >= 3:
                print("---map_project_pom-----------------------------------------------------------------------------------------------------------")
                print(map_project_pom)
                print("---map_project_path-----------------------------------------------------------------------------------------------------------")
                print(map_project_path)
            map_project_parents = build_pom_hierarchy(map_project_path)
            if cfg.verbosity >= 3:
                print("---map_project_parents-----------------------------------------------------------------------------------------------------------")
                print(map_project_parents)
            map_project_pom_fixed = update_libraries_in_poms(cfg, map_project_pom, map_project_parents, map2_lib_version_projects_main, central_libs)
            if cfg.fix >= 2:
                write_poms(map_project_pom_fixed, map_project_path)

    elif cfg.type == FIX_PARENTS_FROM_REPO:
        map2_project_anc_versions_main = maven_ancestors(cfg)
        if cfg.verbosity >= 1:
            print("---map2_project_anc_versions_main-----------------------------------------------------------------------------------------------------------")
            print(map2_project_anc_versions_main)

        map2_anc_version_projects_main = merge_libs(cfg, map2_project_anc_versions_main)
        if cfg.verbosity >= 3:
            print("---map2_anc_version_projects_main-----------------------------------------------------------------------------------------------------------")
            print(map2_anc_version_projects_main)
        central_ancs = {}
        for anc in map2_anc_version_projects_main.keys():
            latest_version = find_latest_maven_central(cfg…