From 50035ebff80d13205b3a4060025ef7dfc2612015 Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Sun, 10 Mar 2019 16:51:25 -0400 Subject: [PATCH 1/3] Centralize binary decision into cnd Project --- pros/conductor/project/__init__.py | 51 ++++++++++++++++++++++++++++ pros/serial/devices/vex/v5_device.py | 48 +++++++++----------------- 2 files changed, 67 insertions(+), 32 deletions(-) diff --git a/pros/conductor/project/__init__.py b/pros/conductor/project/__init__.py index f384d73d..9dd29ca3 100644 --- a/pros/conductor/project/__init__.py +++ b/pros/conductor/project/__init__.py @@ -203,6 +203,57 @@ def output(self): return self.__dict__['output'] return 'bin/output.bin' + @property + def binaries(self) -> List[Union[Path, Tuple[int, Path]]]: + if 'kernel' in self.templates: + if self.target == 'cortex': + return [Path(self.templates['kernel'].metadata['output'])] + elif self.target == 'v5': + if 'hot_output' in self.templates['kernel'].metadata and \ + 'cold_output' in self.templates['kernel'].metadata: + use_hot_cold = False + monolith_path = self.location.joinpath(self.output) + hot_path = self.location.joinpath(self.templates['kernel'].metadata['hot_output']) + cold_path = self.location.joinpath(self.templates['kernel'].metadata['cold_output']) + if hot_path.exists() and cold_path.exists(): + logger(__name__).debug(f'Hot and cold files exist! ({hot_path}; {cold_path})') + if monolith_path.exists(): + monolith_mtime = monolith_path.stat().st_mtime + hot_mtime = hot_path.stat().st_mtime + logger(__name__).debug(f'Monolith last modified: {monolith_mtime}') + logger(__name__).debug(f'Hot last modified: {hot_mtime}') + if hot_mtime > monolith_mtime: + use_hot_cold = True + logger(__name__).debug('Hot file is newer than monolith!') + else: + use_hot_cold = True + if use_hot_cold: + return \ + [ + ( + int(self.templates['kernel'].metadata['hot_addr']), + Path(self.templates['kernel'].metadata['hot_output']) + ), + ( + int(self.templates['kernel'].metadata['cold_addr']), + Path(self.templates['kernel'].metadata['cold_output']) + ) + ] + return [Path(self.templates['kernel'].metadata['output'])] + else: + raise ValueError(f'Unsupported target: "{self.target}"') + + @property + def elfs(self) -> List[Union[Path, Tuple[int, Path]]]: + return \ + [ + b.with_suffix('.elf') + if isinstance(b, Path) + else (b[0], b[1].with_suffix('.elf')) + for b + in self.binaries + ] + def make(self, build_args: List[str]): import subprocess env = os.environ.copy() diff --git a/pros/serial/devices/vex/v5_device.py b/pros/serial/devices/vex/v5_device.py index 8e7dbcb4..f79b2b80 100644 --- a/pros/serial/devices/vex/v5_device.py +++ b/pros/serial/devices/vex/v5_device.py @@ -227,39 +227,23 @@ def generate_cold_hash(self, project: Project, extra: dict): def upload_project(self, project: Project, **kwargs): assert project.target == 'v5' - monolith_path = project.location.joinpath(project.output) - if monolith_path.exists(): - logger(__name__).debug(f'Monolith exists! ({monolith_path})') - if 'hot_output' in project.templates['kernel'].metadata and \ - 'cold_output' in project.templates['kernel'].metadata: - hot_path = project.location.joinpath(project.templates['kernel'].metadata['hot_output']) - cold_path = project.location.joinpath(project.templates['kernel'].metadata['cold_output']) - upload_hot_cold = False - if hot_path.exists() and cold_path.exists(): - logger(__name__).debug(f'Hot and cold files exist! ({hot_path}; {cold_path})') - if monolith_path.exists(): - monolith_mtime = monolith_path.stat().st_mtime - hot_mtime = hot_path.stat().st_mtime - logger(__name__).debug(f'Monolith last modified: {monolith_mtime}') - logger(__name__).debug(f'Hot last modified: {hot_mtime}') - if hot_mtime > monolith_mtime: - upload_hot_cold = True - logger(__name__).debug('Hot file is newer than monolith!') - else: - upload_hot_cold = True - if upload_hot_cold: - with hot_path.open(mode='rb') as hot: - with cold_path.open(mode='rb') as cold: - kwargs['linked_file'] = cold - kwargs['linked_remote_name'] = self.generate_cold_hash(project, {}) - kwargs['linked_file_addr'] = int( - project.templates['kernel'].metadata.get('cold_addr', 0x03800000)) - kwargs['addr'] = int(project.templates['kernel'].metadata.get('hot_addr', 0x07800000)) - return self.write_program(hot, **kwargs) - if not monolith_path.exists(): + binaries = project.binaries + bin = None + if len(binaries) == 1: + if isinstance(binaries[0], tuple): + kwargs['addr'] = binaries[0][0] + bin = binaries[0][1] + else: + bin = binaries[0] + elif len(binaries) == 2: + kwargs['linked_file'] = binaries[1][1].open(mode='rb') + kwargs['linked_remote_name'] = self.generate_cold_hash(project, {}) + kwargs['linked_file_addr'] = binaries[1][0] + kwargs['addr'] = binaries[1][0] + bin = binaries[0][1] + if bin is None or not bin.exists(): raise ui.dont_send(Exception('No output files were found! Have you built your project?')) - with monolith_path.open(mode='rb') as pf: - return self.write_program(pf, **kwargs) + return self.write_program(bin.open(mode='rb'), **kwargs) def generate_ini_file(self, remote_name: str = None, slot: int = 0, ini: ConfigParser = None, **kwargs): project_ini = ConfigParser() From 651f85b0335b37a97e9ed423426964e29e6077f1 Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Sun, 10 Mar 2019 20:58:14 -0400 Subject: [PATCH 2/3] WIP on StackTraceModal --- pros/cli/test.py | 6 +- .../ui/interactive/components/__init__.py | 4 +- .../common/ui/interactive/components/input.py | 5 + pros/common/utils.py | 17 ++ pros/conductor/interactive/StackTraceModal.py | 182 ++++++++++++++++++ pros/conductor/project/__init__.py | 11 +- requirements.txt | 1 + 7 files changed, 212 insertions(+), 14 deletions(-) create mode 100644 pros/conductor/interactive/StackTraceModal.py diff --git a/pros/cli/test.py b/pros/cli/test.py index f19ac9a8..eabbb8e0 100644 --- a/pros/cli/test.py +++ b/pros/cli/test.py @@ -1,4 +1,5 @@ from pros.common.ui.interactive.renderers import MachineOutputRenderer +from pros.conductor import Project from pros.conductor.interactive.NewProjectModal import NewProjectModal from .common import default_options, pros_root @@ -12,7 +13,6 @@ def test_cli(): @test_cli.command() @default_options def test(): - app = NewProjectModal() + from pros.conductor.interactive.StackTraceModal import StackTraceModal + app = StackTraceModal() MachineOutputRenderer(app).run() - - # ui.confirm('Hey') diff --git a/pros/common/ui/interactive/components/__init__.py b/pros/common/ui/interactive/components/__init__.py index e470f931..c0f75e4b 100644 --- a/pros/common/ui/interactive/components/__init__.py +++ b/pros/common/ui/interactive/components/__init__.py @@ -2,9 +2,9 @@ from .checkbox import Checkbox from .component import Component from .container import Container -from .input import DirectorySelector, FileSelector, InputBox +from .input import DirectorySelector, FileSelector, InputBox, TextEditor from .input_groups import ButtonGroup, DropDownBox from .label import Label, Spinner, VerbatimLabel __all__ = ['Component', 'Button', 'Container', 'InputBox', 'ButtonGroup', 'DropDownBox', 'Label', - 'DirectorySelector', 'FileSelector', 'Checkbox', 'Spinner', 'VerbatimLabel'] + 'DirectorySelector', 'FileSelector', 'Checkbox', 'Spinner', 'VerbatimLabel', 'TextEditor'] diff --git a/pros/common/ui/interactive/components/input.py b/pros/common/ui/interactive/components/input.py index 8d35b5e8..ec99dee6 100644 --- a/pros/common/ui/interactive/components/input.py +++ b/pros/common/ui/interactive/components/input.py @@ -28,3 +28,8 @@ class FileSelector(InputBox[P], Generic[P]): class DirectorySelector(InputBox[P], Generic[P]): pass + + +# For a larger InputBox intended for multiline editing +class TextEditor(InputBox[P], Generic[P]): + pass diff --git a/pros/common/utils.py b/pros/common/utils.py index f825267c..6cb98e5b 100644 --- a/pros/common/utils.py +++ b/pros/common/utils.py @@ -3,6 +3,7 @@ import os.path import sys from functools import lru_cache, wraps +from pathlib import Path from typing import * import click @@ -145,3 +146,19 @@ def download_file(url: str, ext: Optional[str] = None, desc: Optional[str] = Non def dont_send(e: Exception): e.sentry = False return e + + +def find_executable(file: str, suggested_locations: List[Path] = None) -> str: + if os.name == 'nt' and not file.endswith('.exe'): + file += '.exe' + if not suggested_locations: + suggested_locations = [] + if os.environ.get('PROS_TOOLCHAIN'): + suggested_locations.append(Path(os.environ.get('PROS_TOOLCHAIN')).joinpath('bin')) + for p in suggested_locations: + if p.joinpath(file).exists(): + return str(p.joinpath(file)) + for p in os.environ.get('PATH').split(os.pathsep): + if Path(p).joinpath(file).exists(): + return str(Path(p).joinpath(file)) + return file diff --git a/pros/conductor/interactive/StackTraceModal.py b/pros/conductor/interactive/StackTraceModal.py new file mode 100644 index 00000000..d9de89ac --- /dev/null +++ b/pros/conductor/interactive/StackTraceModal.py @@ -0,0 +1,182 @@ +import subprocess +from pathlib import Path +from typing import * + +from click import Context, get_current_context + +from pros.common import ui, utils +from pros.common.ui.interactive import application, components, parameters +from pros.common.ui.interactive.components import Component +from pros.conductor import Project +from pros.conductor.interactive import ExistingProjectParameter + + +class _ElfReporter(object): + def __init__(self, elfs, root=None): + if root is None: + root = Path('.') + # if we don't have an address for the ELF, assume it's 0xffff_ffff for now + self.elfs = [root.joinpath(e[1]) if isinstance(e, tuple) else root.joinpath(e) for e in elfs] + from elftools.elf.elffile import ELFFile + ui.echo(root) + ui.echo(self.elfs) + self.elf_files = [ELFFile(e.open(mode='rb')) for e in self.elfs] + self._timestamp = None + + self._addr2line = utils.find_executable('arm-none-eabi-addr2line') or utils.find_executable('addr2line') + + @property + def timestamp(self) -> str: + if self._timestamp is not None: + return self._timestamp + from elftools.common.exceptions import ELFError + from elftools.elf.sections import SymbolTableSection + for elf in self.elf_files: + try: + for symtab in elf.iter_sections(): + if not isinstance(symtab, SymbolTableSection): + continue + syms = symtab.get_symbol_by_name('_PROS_COMPILE_TIMESTAMP') + if not syms or len(syms) != 1: + continue + sym_entry = syms[0].entry + ptr_sec = elf.get_section(sym_entry['st_shndx']) + off = sym_entry['st_value'] - ptr_sec.header['sh_addr'] + from struct import unpack + str_addr = unpack(' List[str]: + start = dump.find('BEGIN STACK TRACE') + if start == -1: + start = 0 + else: + start += len('BEGIN STACK TRACE') + + end = dump.find('END OF TRACE') + if end == -1: + end = len(dump) + dump = dump[start:end] + ui.echo(dump) + return list(filter(bool, [self.addr2line(line.strip()) for line in dump.splitlines()])) + + + +class StackTraceModal(application.Modal[None]): + @property + def processing_project(self): + return self._processing_project + + @processing_project.setter + def processing_project(self, value: bool): + self._processing_project = bool(value) + self.redraw() + + @property + def processing_dump(self): + return self._processing_dump + + @processing_dump.setter + def processing_dump(self, value: bool): + self._processing_dump = bool(value) + self.redraw() + + def __init__(self, ctx: Optional[Context] = None, project: Optional[Project] = None): + super().__init__('Stack Trace') + self.click_ctx: Context = ctx or get_current_context() + self.project: Optional[Project] = project + + self.project_path = ExistingProjectParameter( + str(project.location) if project else str(Path('~', 'My PROS Project').expanduser()) + ) + + self.input: parameters.Parameter[str] = parameters.Parameter('') + self.report: Optional[_ElfReporter] = None + self.sources: str = "" + self.timestamp: str = "" + + self.detail_collapsed = parameters.BooleanParameter(False) + self._processing_project: bool = False + + self._processing_dump: bool = False + self.dump = '' + + cb = self.project_path.on_changed(self.project_changed, asynchronous=True) + if self.project_path.is_valid(): + cb(self.project_path) + + self.input.on_changed(self.input_changed, asynchronous=True) + + def project_changed(self, new_project: ExistingProjectParameter): + self.processing_project = True + self.project = Project(new_project.value) + elfs = self.project.elfs + self.report = _ElfReporter(elfs, root=self.project.path) + if len(elfs) == 0: + self.sources = '' + elif isinstance(elfs[0], tuple): + self.sources = '\n'.join([str(self.project.location.joinpath(e[1])) for e in elfs]) + else: + self.sources = '\n'.join([str(self.project.location.joinpath(e)) for e in elfs]) + self.processing_project = False + self.input_changed(self.input) + + def confirm(self, *args, **kwargs): + pass + + def input_changed(self, new_input: parameters.Parameter[str]): + if self.report is None or not new_input.value: + return + self.processing_dump = True + self.dump = '\n'.join(self.report.parse_dump(new_input.value.strip())) + self.processing_dump = False + + def build_detail_container(self) -> Generator[Component, None, None]: + if self.sources: + yield components.Label('Sources:') + yield components.VerbatimLabel(self.sources) + if self.report and self.report.timestamp: + yield components.Label(f'Compiled: {self.report.timestamp}') + + def build(self) -> Generator[Component, None, None]: + yield components.DirectorySelector('Project Directory', self.project_path) + if self.processing_project: + yield components.Spinner() + else: + yield components.Container(*self.build_detail_container(), title='Details', collapsed=self.detail_collapsed) + yield components.Label('Paste data abort dump below') + yield components.TextEditor('', self.input) + if self.processing_dump or self.processing_project: + yield components.Spinner() + elif self.dump: + yield components.Label('Stack trace') + yield components.VerbatimLabel(self.dump) diff --git a/pros/conductor/project/__init__.py b/pros/conductor/project/__init__.py index 9dd29ca3..e9c1abd4 100644 --- a/pros/conductor/project/__init__.py +++ b/pros/conductor/project/__init__.py @@ -261,11 +261,7 @@ def make(self, build_args: List[str]): if os.environ.get('PROS_TOOLCHAIN'): env['PATH'] = os.path.join(os.environ.get('PROS_TOOLCHAIN'), 'bin') + os.pathsep + env['PATH'] - # call make.exe if on Windows - if os.name == 'nt' and os.environ.get('PROS_TOOLCHAIN'): - make_cmd = os.path.join(os.environ.get('PROS_TOOLCHAIN'), 'bin', 'make.exe') - else: - make_cmd = 'make' + make_cmd = utils.find_executable('make') stdout_pipe = EchoPipe() stderr_pipe = EchoPipe(err=True) process = subprocess.Popen(executable=make_cmd, args=[make_cmd, *build_args], cwd=self.directory, env=env, @@ -327,10 +323,7 @@ def libscanbuild_capture(args: argparse.Namespace) -> Tuple[int, Iterable[Compil return exit_code, iter(set(current)) # call make.exe if on Windows - if os.name == 'nt' and os.environ.get('PROS_TOOLCHAIN'): - make_cmd = os.path.join(os.environ.get('PROS_TOOLCHAIN'), 'bin', 'make.exe') - else: - make_cmd = 'make' + make_cmd = utils.find_executable('make') args = create_intercept_parser().parse_args( ['--override-compiler', '--use-cc', 'arm-none-eabi-gcc', '--use-c++', 'arm-none-eabi-g++', make_cmd, *build_args, diff --git a/requirements.txt b/requirements.txt index 7a6a2e0b..419457f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ rfc6266-parser sentry-sdk observable pypng +pyelftools From 1d2d901c1ba8cf098f86126e37af45159cf15c3f Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Sun, 10 Mar 2019 21:01:03 -0400 Subject: [PATCH 3/3] Fix address user program is uploaded to --- pros/serial/devices/vex/v5_device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pros/serial/devices/vex/v5_device.py b/pros/serial/devices/vex/v5_device.py index f79b2b80..4a184e27 100644 --- a/pros/serial/devices/vex/v5_device.py +++ b/pros/serial/devices/vex/v5_device.py @@ -239,7 +239,7 @@ def upload_project(self, project: Project, **kwargs): kwargs['linked_file'] = binaries[1][1].open(mode='rb') kwargs['linked_remote_name'] = self.generate_cold_hash(project, {}) kwargs['linked_file_addr'] = binaries[1][0] - kwargs['addr'] = binaries[1][0] + kwargs['addr'] = binaries[0][0] bin = binaries[0][1] if bin is None or not bin.exists(): raise ui.dont_send(Exception('No output files were found! Have you built your project?'))