Skip to content

🎨优化工具依赖配置读取逻辑;优化LD_LIBRARY_PATH环境变量处理;废弃DEFAULT_TOOL_BASE_DIR #561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cyw3 merged 1 commit into from
Aug 24, 2022
Merged

🎨优化工具依赖配置读取逻辑;优化LD_LIBRARY_PATH环境变量处理;废弃DEFAULT_TOOL_BASE_DIR #561

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 40 additions & 8 deletions client/node/toolloader/loadconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,8 @@ def __update_tool_names(self, tool_names, task_list):
tool_names.remove(c_name)
return tool_names

def read_tool_config_from_ini_file(self, tool_names=None, custom_tools=None, task_list=None, config_all_tools=False, include_common=True):
def read_tool_config_from_ini_file(self, tool_names=None, custom_tools=None, task_list=None, config_all_tools=False,
include_common=True):
"""
从ini文件中读取工具配置
"""
Expand Down Expand Up @@ -194,6 +195,7 @@ def read_tool_config_from_ini_file(self, tool_names=None, custom_tools=None, tas
"env_path": env_path_section, # env_path_section中现在已经没有key为"PATH"的元素,可以作为编译工具的env_path
"env_value": env_value_section
}
self.__format_to_fullpath(compile_config)
config_dict["compile_config"] = compile_config
return config_dict
else:
Expand Down Expand Up @@ -236,6 +238,25 @@ def __format_config(self, tool_cfg, env_path_section, env_value_section):
tool_cfg["env_value"] = self.__str_to_dict(tool_cfg["env_value"], env_value_section)
tool_cfg["tool_url"] = self.__str_to_list(tool_cfg["tool_url"])
tool_cfg["path"] = self.__str_to_list(tool_cfg["path"], uniq=False)
self.__format_to_fullpath(tool_cfg)

def __format_to_fullpath(self, tool_cfg):
# 将路径类型的环境变量（env_path）中的相对路径转换成绝对路径
for env_name, rel_path in tool_cfg["env_path"].items():
if "PATH" == env_name: # PATH应该单独放在path字段中，如果放在env_path中，忽略，避免影响和覆盖原有PATH变量
continue
full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
tool_cfg["env_path"][env_name] = full_path

# 将PATH环境变量的相对路径转换成绝对路径
path_env = []
for rel_path in tool_cfg["path"]:
if "$" in rel_path or "%" in rel_path: # 带变量的环境变量，已经是全路径
full_path = rel_path
else:
full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
path_env.append(full_path)
tool_cfg["path"] = path_env

def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
"""
Expand Down Expand Up @@ -266,7 +287,7 @@ def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
config_dict = {}
already_config_tools = [] # 记录已经读取到配置的工具
for task_config in task_list:
task_name = task_config["task_name"]
task_name = task_config.get("task_name")
if tool_names and task_name not in tool_names:
continue
task_params = task_config.get("task_params", {})
Expand All @@ -282,29 +303,40 @@ def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
for tool_lib in tool_libs:
lib_support_os = tool_lib.get("os", [])
if self._os_type in lib_support_os:
# 使用新的结构存储环境变量，避免修改原有的任务参数
new_lib_envs = {}
lib_envs = tool_lib.get("envs", {})
if not lib_envs: # 可能会传空list，这里判空，避免后面格式出错
lib_envs = {}
new_lib_envs = {}

scm_url = tool_lib.get("scm_url")
lib_dir_name = scm_url.split('/')[-1].strip().replace(".git", "")
# 环境变量中的$ROOT_DIR替换为目录名，后续加载环境变量时会拼接为全路径
lib_dir_path = os.path.join(settings.TOOL_BASE_DIR, lib_dir_name)

# 将环境变量中的$ROOT_DIR替换为实际路径，重新保存到new_lib_envs
for key, value in lib_envs.items():
if "$ROOT_DIR" in value:
lib_envs[key] = value.replace("$ROOT_DIR", lib_dir_name)
new_lib_envs[key] = value.replace("$ROOT_DIR", lib_dir_path)
else:
new_lib_envs[key] = value

# PATH和其他环境变量分开处理
path_envs = []
other_envs = {}
for env_name, env_value in lib_envs.items():
for env_name, env_value in new_lib_envs.items():
if env_name == "PATH":
path_envs = self.__str_to_list(env_value)
else:
other_envs[env_name] = env_value
# 根据英文分号拆分后，再环境变量分隔符拼接到一起
value_list = self.__str_to_list(env_value)
value_format = os.pathsep.join(value_list)
other_envs[env_name] = value_format
lib_config = {
"tool_url": scm_url,
"scm_type": tool_lib.get("scm_type"),
"auth_info": tool_lib.get("auth_info"),
"path": path_envs,
"env_path": lib_envs,
"env_path": other_envs,
"env_value": {}
}
lib_name = tool_lib.get("name")
Expand Down
2 changes: 0 additions & 2 deletions client/settings/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,6 @@
# ========================
# 默认从Git拉取工具；如果使用本地工具，可以在config.ini中配置该值为True，将不自动拉取内置工具和配置文件
USE_LOCAL_TOOL = False
# 默认的工具目录，即puppy根目录下的data/tools目录
DEFAULT_TOOL_BASE_DIR = join(TCA_BASE_DIR, "tools")
# 扫描工具目录
TOOL_BASE_DIR = join(TCA_BASE_DIR, "tools")
# 扫描工具配置文件地址，需要在config.ini中配置
Expand Down
58 changes: 32 additions & 26 deletions client/util/envset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@
import logging
import platform

from node.app import settings
from node.toolloader.loadconfig import ConfigLoader
from util.textutil import StringMgr

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -67,6 +65,7 @@ def __update_os_environ(self, envs, task_mode = True):
os.environ.update(new_envs)
else:
for env_name, env_value in envs.items():
logger.debug(f">>> set env {env_name}={os.path.expandvars(env_value)}")
os.environ.update({env_name: os.path.expandvars(env_value)})

def set_tool_env(self, tool_config):
Expand All @@ -88,17 +87,13 @@ def set_tool_env(self, tool_config):
"""
tool_envs = {}

path_evn = []
path_env = []
for tool_name, tool_params in tool_config.items():
# 添加路径类型的环境变量
for env_name, rel_path in tool_params["env_path"].items():
for env_name, full_path in tool_params["env_path"].items():
if "PATH" == env_name: # PATH应该单独放在path字段中，如果放在env_path中，忽略，避免影响和覆盖原有PATH变量
continue
if env_name not in tool_envs:
full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
# 如果工具目录不存在，重定向到默认的工具目录下
if not os.path.exists(full_path):
full_path = os.path.join(settings.DEFAULT_TOOL_BASE_DIR, rel_path)
tool_envs[env_name] = full_path

# 添加值类型的环境变量
Expand All @@ -108,21 +103,17 @@ def set_tool_env(self, tool_config):

# 将PATH的环境变量放到一个list中
for rel_path in tool_params["path"]:
if rel_path not in path_evn:
path_evn.append(rel_path)
if rel_path not in path_env:
path_env.append(rel_path)

# 添加PATH环境变量
path_str = os.environ["PATH"] # 先读取系统PATH环境变量
path_list = path_str.split(os.pathsep)
for rel_path in path_evn:
if "$" in rel_path or "%" in rel_path: # 带变量的环境变量，已经是全路径
full_path = rel_path
else:
full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
# 如果工具目录不存在，重定向到默认的工具目录下
if not os.path.exists(full_path):
full_path = os.path.join(settings.DEFAULT_TOOL_BASE_DIR, rel_path)
# if absolute_path not in path_list: # path不要去重,会影响加载顺序
path_str = os.getenv("PATH") # 先读取系统PATH环境变量
if path_str:
path_list = path_str.split(os.pathsep)
else: # 增加判空，防止读取不到的情况
path_list = []
for full_path in path_env:
# path不要去重,会影响加载顺序
path_list = [full_path] + path_list # 后添加的放在前面,优先搜索
tool_envs["PATH"] = os.pathsep.join(path_list)

Expand Down Expand Up @@ -156,6 +147,7 @@ def set_task_env(self, task_params):
if task_envs:
# 设置到进程环境变量中
self.__update_os_environ(task_envs, True)
# logger.debug('设置任务环境变量::\n%s' % '\n'.join(['%s=%s' % (key, value) for key, value in task_envs.items()]))
EnvSetting.env_setting_init()

def get_origin_env(self, os_env=None):
Expand All @@ -175,17 +167,31 @@ def get_origin_env(self, os_env=None):
env = os_env
else:
env = dict(os.environ)

# 判断是否是pyinstaller包状态, 若是源码状态，则不处理
if len(sys.argv) > 0 and sys.argv[0].endswith(".py"):
return env

# Linux
if sys.platform == "linux" or sys.platform == "linux2":
lp_key = 'LD_LIBRARY_PATH'
lp_orig = env.get(lp_key + '_ORIG')
if lp_orig is not None:
env[lp_key] = lp_orig
else:
env.pop(lp_key, None)
# 收集需要的lp环境变量
wanted_env_list = []
# 收集现有进程的lp环境变量，剔除掉pyinstaller添加的临时运行目录，则为需要传递给子进程的环境变量
cur_lp_value = env.get(lp_key)
if cur_lp_value:
cur_lp_list = cur_lp_value.split(os.pathsep)
for lp_value in cur_lp_list:
# 当前进程的lp环境变量，剔除掉pyinstaller添加的临时运行目录，其他的为需要的
if not lp_value.startswith("/tmp/_MEI"):
wanted_env_list.append(lp_value)
if wanted_env_list: # 现有的lp已经包含_ORIG
env[lp_key] = os.pathsep.join(wanted_env_list)
logger.debug(f"{lp_key}={cur_lp_value}, change to {env[lp_key]}")
else:
env.pop(lp_key, None)
logger.debug(f"{lp_key}={cur_lp_value}, delete it.")

# Mac:
# 由于Mac下二进制包无法检测环境中环境变量原来是否有DYLD_LIBRARY_PATH，加上目前没有发现有项目遇到类似问题
# 这里暂不做处理，继续观察
Expand Down