1#@!#!123s

: / opt / alt / python37 / lib / python3.7 / site-packages / clselect / clselectpython /
Filename : apps_manager.py
back
# coding=utf-8

# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import logging
import os
import pwd
import json

from typing import AnyStr, Optional  # NOQA
from future.utils import iteritems

import secureio

from clselect.utils import get_abs_rel, run_process_in_cagefs
from clcommon.clpwd import drop_privileges
from clselect.baseclselect import (
    BaseSelectorError,
    MissingVirtualenvError,
    MissingAppRootError,
)

from clselect.clpassenger import WSGI_PATH
from clselect import ClSelectExcept, clpassenger, clselectctl
from clselect.baseclselect.apps_manager import BaseApplicationsManager
from clselect.clselectpython.python_manager import PythonManager


class PythonAppFormatVersion(object):
    """
    Class represents possible python application versions.
    """
    # application that was created in old selector
    # does not support version change and env variables
    LEGACY = 1
    # new application, supports all selector features
    STABLE = 2


class PythonVenvFormatVersion(object):
    """
    Class represents possible python application venv path versions.
    """
    # venv that was created in old selector
    # and was transformed like 'app/test_1' -> 'app_test__1'
    LEGACY = 1
    # any application (old & new) that does not contain
    # replacements in venv path
    STABLE = 2


class ApplicationsManager(BaseApplicationsManager):
    _USER_CONFIG = '.cl.selector/python-selector.json'
    _IMPORT_STATUS_ADMIN = '/var/lve/.py.migration.status'
    _IMPORT_STATUS_USER = '.cl.selector/.py.migration.status'
    _IMPORT_LOG_FILE = '/var/log/cloudlinux/py-selector/app_imports.log'
    _IMPORT_REQUIREMENTS_FILE = 'imported_pip_requirements.txt'
    INTERPRETER = 'python'
    VENV_DIR = 'virtualenv'
    BINARY_NAME = 'python'

    def __init__(self):
        super(ApplicationsManager, self).__init__(PythonManager())

    # TODO: replace with real implementation (LVEMAN-1324)
    def _find_config_files(self, user_name, app_directory, patterns=()):

        return super(ApplicationsManager, self)._find_config_files(user_name, app_directory, patterns=patterns)

    @staticmethod
    def get_interpreter_specific_passenger_lines(binary_path, app_config):
        """
        Return lines for htaccess that are specific to the python interpreter
        :param binary_path: path to the environment's python binary
        :param app_config: application's config dictionary
        """
        specific_lines = ['PassengerPython "{}"\n'.format(binary_path)]
        return specific_lines

    def _get_legacy_applications(self, user, logger):
        """
        Search server for old applications (created in old python selector)
        and add them to new config.
        :param user: user to search applications
        :param logger: logger to write messages to
        :return: tuple(list[applications], list[errors])
        """
        from clselect.clselectctlpython import WSGI_PATTERN
        applications = {}
        errors = {}

        user_config_data = self.get_user_config_data(user)
        for directory, data in iteritems(clpassenger.summary(user)):
            if data['interpreter'] != self.INTERPRETER:
                continue
            # noinspection PyBroadException
            try:
                app_wsgi = os.path.join(data['directory'], clpassenger.WSGI_PATH)
                # skip app if wsgi file was not found
                if not os.path.isfile(app_wsgi):
                    logger.warning('Application %s was skipped during import, because wsgi file was not found',
                                   directory)
                    continue
                with open(app_wsgi) as f:
                    wsgi_conf = f.read()
                match = WSGI_PATTERN.search(wsgi_conf)
                if match:
                    groupdict = match.groupdict()
                    wsgi_path = groupdict['script']
                    callable_object = groupdict['callable']
                else:
                    wsgi_path = callable_object = ''

                py_version = os.path.basename(os.path.dirname(
                    os.path.dirname(data['binary'])))
                domain = data['domains'][0]

                # LVEMAN-1502. if application already present in apps list use its app_version
                try:
                    app_ver = user_config_data[directory][u'app_version']
                except KeyError:
                    app_ver = PythonAppFormatVersion.LEGACY

                app_info = {
                    u'python_version': py_version,
                    u'app_version': app_ver,
                    u'domain': domain,
                    u'app_uri': data['alias'],
                    u'app_status': 'started',
                    u'startup_file': wsgi_path or WSGI_PATH,
                    u'config_files': [],
                    u'env_vars': {},
                    u'entry_point': callable_object or 'application'
                }

                applications[directory] = app_info
            except Exception:
                logger.exception('Unable to import application %s', directory)
                errors.setdefault(directory, []).append(
                    'Unexpected issue during application "%s" import. '
                    'Your python app will work as before, but you wo\'t be able '
                    'to control it from Python Selector UI. Please, ask your system '
                    'administrator to contact CloudLinux support '
                    'to resolve this issue.' % directory
                )
                continue

        return applications, errors

    def _setup_import_logger(self):
        """
        Setup logger for application import.
        """
        app_logger = logging.getLogger('import_apps')
        app_logger.setLevel(logging.DEBUG)
        fh = logging.FileHandler(self._IMPORT_LOG_FILE)
        fh.formatter = logging.Formatter(
            '[%(levelname)s | %(asctime)s]: %(message)s')
        cl = logging.StreamHandler()
        cl.setLevel(logging.INFO)
        app_logger.addHandler(fh)
        # app_logger.addHandler(cl)
        return app_logger

    def import_legacy_applications_to_config(self, user=None):
        """
        Scan users for legacy applications (created by old python selector)
        and import them into config file. Done automatically by spec file.
        """
        app_logger = self._setup_import_logger()

        users_info = self.get_users_dict(user)
        logging.debug('Start applications import for users %s', list(users_info.keys()))
        failed_users = []
        skipped_users_count = 0
        apps_errors = None

        for user, pw in iteritems(users_info):
            # Skip user if '.cl.selector' directory absent in his homedir
            cl_selector_dir = os.path.join(pw.pw_dir, '.cl.selector')
            if not os.path.isdir(cl_selector_dir):
                skipped_users_count += 1
                app_logger.warning('User %s is skipped due to %s directory absent' % (user, cl_selector_dir))
                continue
            app_logger.info('Importing user %s', user)

            try:
                # disable quota here, because clpassanger.summary creates htaccess_cache file
                # see clpassenger._summary for details. Will be fixed in LVEMAN-1524
                with drop_privileges(user), secureio.disable_quota():
                    try:
                        config = self.read_user_selector_config_json(
                            pw.pw_dir, pw.pw_uid, pw.pw_gid)
                    except BaseSelectorError:
                        config = {}

                    # take applications from clpassanger.summary and import them
                    apps, apps_errors = self._get_legacy_applications(user, app_logger)
                    if not apps:
                        continue

                for app_root, app_config in iteritems(apps):
                    if app_root in config:
                        # save fields that were probably set before
                        app_config['config_files'] = config[app_root]['config_files']
                        app_config['env_vars'] = config[app_root]['env_vars']

                    # generate pip requirements file for each legacy app
                    self.generate_pip_requirements(user, app_root, app_logger)
                    requirements_file = os.path.join(pw.pw_dir, app_root, self._IMPORT_REQUIREMENTS_FILE)
                    if os.path.isfile(requirements_file) and \
                            self._IMPORT_REQUIREMENTS_FILE not in app_config['config_files']:
                        # add this newly generated pip requirements file to config files
                        app_config['config_files'].append(self._IMPORT_REQUIREMENTS_FILE)

                    config[app_root] = app_config

                # case when hoster downgraded package for some time
                # and user destroyed some of applications
                # we should remove them from config file
                destroyed_apps = set(config.keys()) - set(apps.keys())
                for app_root in destroyed_apps:
                    config.pop(app_root)

                with drop_privileges(user), secureio.disable_quota():
                    self.write_full_user_config_data(user, config)
                    self._set_user_legacy_import_status(pw, is_import_failed=False, apps_errors=apps_errors)
            except Exception as e:
                import_error = 'Unable to import user `%s` to new python selector. ' + \
                               'Already created applications work as before, ' + \
                               'but user won\'t be able to manage them. Error is: `%s`'
                app_logger.exception(import_error, user, e)
                with drop_privileges(user), secureio.disable_quota():
                    self._set_user_legacy_import_status(pw, is_import_failed=True, apps_errors=apps_errors)
                failed_users.append(user)

        self._set_admin_legacy_import_status(failed_users)
        if skipped_users_count != 0:
            app_logger.warning('Some users skipped... see import log')

    def _get_legacy_import_status(self, user=None):
        """
        Read import log which contains information about
        failed users or apps depending on current user.
        """
        if user is None:
            if not os.path.exists(self._IMPORT_STATUS_ADMIN):
                return None

            with open(self._IMPORT_STATUS_ADMIN) as f:
                return json.load(f)
        else:
            user_pw = pwd.getpwnam(user)
            marker = os.path.join(user_pw.pw_dir, self._IMPORT_STATUS_USER)
            if not os.path.isfile(marker):
                return

            with open(marker) as f:
                return json.load(f)

    def _set_user_legacy_import_status(self, user_pw, is_import_failed, apps_errors):
        """
        Save information that some applications were not imported automatically
        to show warning for user in the future.
        """
        if not os.path.exists(os.path.join(user_pw.pw_dir, '.cl.selector')):
            return
        try:
            marker = os.path.join(user_pw.pw_dir, self._IMPORT_STATUS_USER)
            secureio.write_file_via_tempfile(json.dumps({
                'is_import_failed': is_import_failed,
                'apps_lookup_failed': apps_errors
            }), marker, 0o600)
        except (IOError, OSError):
            # sad, but not critical, go further
            logging.exception('Unable to save migration status file')

    def _set_admin_legacy_import_status(self, failed_users):
        """
        Save information that some users were not imported automatically
        to show warning for admin in the future.
        """
        packed = json.dumps({'failed_users': failed_users})
        secureio.write_file_via_tempfile(packed, self._IMPORT_STATUS_ADMIN, 0o600)

    def _get_admin_legacy_import_warning(self):
        config = self._get_legacy_import_status()

        if config is None or not config['failed_users']:
            return None
        warning_msg = 'Unexpected issue(s) happened during importing python ' \
                      'applications for the following users: "{users}". ' \
                      'Everything will work as before, but listed users wo\'t be able ' \
                      'to control applications from Python Selector UI. ' \
                      'Please, contact CloudLinux support and send them log file located at ' \
                      '`{log_file_path}` to investigate and ' \
                      'resolve this issue. Also you can hide this warning ' \
                      'by deleting `{import_warning_marker}` file' \
                      ''.format(users=','.join(config['failed_users']),
                                log_file_path=self._IMPORT_LOG_FILE,
                                import_warning_marker=self._IMPORT_STATUS_ADMIN)
        return warning_msg

    def _get_user_legacy_import_warning(self, username):
        config = self._get_legacy_import_status(user=username)
        if config is None:
            return None

        what_to_do_msg = \
            'Everything will work as before, but you won\'t be able to control ' \
            'listed applications from Python Selector UI. ' \
            'Please, ask you hoster to contact CloudLinux support ' \
            'to investigate and ' \
            'resolve this issue.\nAlso you can hide this warning ' \
            'by deleting `~/{import_warning_marker}` file.' \
            ''.format(import_warning_marker=self._IMPORT_STATUS_USER)

        if config['is_import_failed']:
            return 'Unexpected issue(s) happened during importing python ' \
                   'applications. ' \
                   '%s' % what_to_do_msg
        elif config['apps_lookup_failed']:
            return 'Unexpected issue(s) happened during importing following python ' \
                   'applications: "%s". ' \
                   '%s' % (','.join(config['apps_lookup_failed']), what_to_do_msg)
        return None

    def _get_legacy_import_warning_or_none(self, user=None):
        if user is None:
            return self._get_admin_legacy_import_warning()
        else:
            return self._get_user_legacy_import_warning(user)

    def migrate_application(self, user, app_root):
        """
        Update environment of specific application to support
        features of new python selector
        """
        from clselect.clselectctlpython import _get_environment
        application = self.get_app_config(user, app_root)
        if application is None:
            raise ClSelectExcept.WrongData("Application %s does not exist" % app_root)

        if application['app_version'] == PythonAppFormatVersion.STABLE:
            raise ClSelectExcept.WrongData(
                "Application %s is already new version "
                "and does not need any updates" % app_root)

        environment = _get_environment(user, app_root, apps_manager=self)
        try:
            environment.configure_environment(auto_restore=True)
        except Exception as e:
            raise ClSelectExcept.WrongData(
                "Unable to migrate application %s. "
                "Error is '%s', everything restored to work as before migration. "
                "Try again later or ask your hoster "
                "to contact CloudLinux support if the issue persists."
                "" % (app_root, e))

        application['app_version'] = PythonAppFormatVersion.STABLE
        self.add_app_to_config(user, app_root, application)

    def get_applications_users_info(self, user=None):
        result = super(ApplicationsManager, self). \
            get_applications_users_info(user)
        warning = self._get_legacy_import_warning_or_none(user)
        if warning is not None:
            result['warning'] = warning
        return result

    def generate_pip_requirements(self, user, app_root, app_logger):
        # type: (AnyStr, AnyStr, logging.Logger) -> Optional[AnyStr]
        """
        Generates requirements file from python apps
        :param user: username
        :param app_root: app root
        :param app_logger: app logger
        """
        app_path, _ = get_abs_rel(user, app_root)
        req_path = os.path.join(app_path, self._IMPORT_REQUIREMENTS_FILE)
        if os.path.exists(req_path):
            return

        from clselect.clselectctlpython import _get_environment
        with drop_privileges(user):
            environment = _get_environment(
                user, app_root, apps_manager=self)

            pip_path = environment.pip()
        user_home = pwd.getpwnam(user).pw_dir
        env_vars = {'HOME': user_home}

        modules = ''
        # need to pass users home as env var directly during running as user
        result = run_process_in_cagefs(user, pip_path, ['freeze', '-l'], env_vars=env_vars)
        if result['returncode'] != 0:
            app_logger.warning('Error during generation pip requirements file. ' + str(result['output']))
            # like in `check_output`
            raise ClSelectExcept.ExternalProgramFailed(result['output'])
        elif not result['failed']:
            modules = result['output']

        # write stdout to file with disabled quota
        with drop_privileges(user), secureio.disable_quota():
            f = open(req_path, 'w')
            f.write(modules)
            f.close()

    def get_app_folders(self, username, app_root, chk_env=True, chk_app_root=True):
        """
        Calculate, check exists and return application folders
        This method does not check that application exists in config.
        :raises: NoSuchUserException, MissingVirtualenvError, MissingAppRootError
        :return: tuple(app_root, app_venv) with absolute paths
        """
        user_home = self._pwd.get_pw_by_name(username).pw_dir
        _, rel_path = get_venv_rel_path(username, app_root)
        app_venv = os.path.join(user_home, rel_path)
        if chk_env and not os.path.exists(app_venv):
            raise MissingVirtualenvError(app_venv)

        app_root = os.path.join(user_home, app_root)
        if chk_app_root and not os.path.exists(app_root):
            raise MissingAppRootError(app_root)

        return app_root, app_venv

    def get_binary_path(self, user, app_root, user_dir, binary_name=None):
        """
        Return a path to the environment's interpreter binary
        Get interpreter path for application
        :param user: owner of the application
        :param app_root: app path relative to user home (app-root)
        :param user_dir: User's home directory
        :param binary_name: name of binary in virtual environemnt (python, npm, node)
        :return: path to interpreter binary in virtual environment
        """
        version = self.get_interpreter_version_for_app(user, app_root)
        if binary_name is None:
            binary_name = self.BINARY_NAME
        _, rel_path = get_venv_rel_path(user, app_root)
        return os.path.join(user_dir, rel_path, version, 'bin', binary_name)


# TODO: we definitely should refactor this one time
def get_venv_rel_path(user, directory, version=None):
    """
    Get path to users VENV relative to home dir.
    Old python selector transforms app_root using get_prefix() method
    before creating venv directory. We should handle both cases:
     - when app is from old selector
     - when app is from new selector
    and return right env directory.
    If both old and new vevns exist we use old one.
    Return tuple with two values:
    - detected version of venv path (LEGACY for
      path with replacements "/" -> "_" & "_" => "__"
      and STABLE in other case)
    - path to venv relative to user's home dir
    """
    _, new_rel_dir = get_abs_rel(user, os.path.join(ApplicationsManager.VENV_DIR, directory))
    old_abs_dir, old_rel_dir = get_abs_rel(
        user, os.path.join(ApplicationsManager.VENV_DIR, clselectctl.get_prefix(directory)))
    if version is None:
        if os.path.exists(old_abs_dir) and directory != clselectctl.get_prefix(directory):
            return PythonVenvFormatVersion.LEGACY, old_rel_dir
        return PythonVenvFormatVersion.STABLE, new_rel_dir
    elif version == PythonVenvFormatVersion.LEGACY:
        return PythonVenvFormatVersion.LEGACY, old_rel_dir
    elif version == PythonVenvFormatVersion.STABLE:
        return PythonVenvFormatVersion.STABLE, new_rel_dir
    else:
        raise ValueError("unknown version `%s`" % version)