A lib/python/qmk/build_targets.py => lib/python/qmk/build_targets.py +211 -0
@@ 0,0 1,211 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+import json
+import shutil
+from typing import List, Union
+from pathlib import Path
+from dotty_dict import dotty, Dotty
+from milc import cli
+from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
+from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
+from qmk.keyboard import keyboard_folder
+from qmk.info import keymap_json
+from qmk.cli.generate.compilation_database import write_compilation_database
+
+
+class BuildTarget:
+ def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
+ self._keyboard = keyboard_folder(keyboard)
+ self._keyboard_safe = self._keyboard.replace('/', '_')
+ self._keymap = keymap
+ self._parallel = 1
+ self._clean = False
+ self._compiledb = False
+ self._target = f'{self._keyboard_safe}_{self.keymap}'
+ self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
+ self._generated_files_path = self._intermediate_output / 'src'
+ self._json = json.to_dict() if isinstance(json, Dotty) else json
+
+ def __str__(self):
+ return f'{self.keyboard}:{self.keymap}'
+
+ def __repr__(self):
+ return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})'
+
+ def configure(self, parallel: int = None, clean: bool = None, compiledb: bool = None) -> None:
+ if parallel is not None:
+ self._parallel = parallel
+ if clean is not None:
+ self._clean = clean
+ if compiledb is not None:
+ self._compiledb = compiledb
+
+ @property
+ def keyboard(self) -> str:
+ return self._keyboard
+
+ @property
+ def keymap(self) -> str:
+ return self._keymap
+
+ @property
+ def json(self) -> dict:
+ if not self._json:
+ self._load_json()
+ if not self._json:
+ return {}
+ return self._json
+
+ @property
+ def dotty(self) -> Dotty:
+ return dotty(self.json)
+
+ def _common_make_args(self, dry_run: bool = False, build_target: str = None):
+ compile_args = [
+ find_make(),
+ *get_make_parallel_args(self._parallel),
+ '-r',
+ '-R',
+ '-f',
+ 'builddefs/build_keyboard.mk',
+ ]
+
+ if not cli.config.general.verbose:
+ compile_args.append('-s')
+
+ verbose = 'true' if cli.config.general.verbose else 'false'
+ color = 'true' if cli.config.general.color else 'false'
+
+ if dry_run:
+ compile_args.append('-n')
+
+ if build_target:
+ compile_args.append(build_target)
+
+ compile_args.extend([
+ f'KEYBOARD={self.keyboard}',
+ f'KEYMAP={self.keymap}',
+ f'KEYBOARD_FILESAFE={self._keyboard_safe}',
+ f'TARGET={self._target}',
+ f'INTERMEDIATE_OUTPUT={self._intermediate_output}',
+ f'VERBOSE={verbose}',
+ f'COLOR={color}',
+ 'SILENT=false',
+ 'QMK_BIN="qmk"',
+ ])
+
+ return compile_args
+
+ def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+ raise NotImplementedError("prepare_build() not implemented in base class")
+
+ def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+ raise NotImplementedError("compile_command() not implemented in base class")
+
+ def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None:
+ self.prepare_build(build_target=build_target, **env_vars)
+ command = self.compile_command(build_target=build_target, dry_run=True, **env_vars)
+ write_compilation_database(command=command, output_path=QMK_FIRMWARE / 'compile_commands.json', skip_clean=skip_clean, **env_vars)
+
+ def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+ if self._clean or self._compiledb:
+ command = [find_make(), "clean"]
+ if dry_run:
+ command.append('-n')
+ cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command))
+ cli.run(command, capture_output=False)
+
+ if self._compiledb and not dry_run:
+ self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars)
+
+ self.prepare_build(build_target=build_target, dry_run=dry_run, **env_vars)
+ command = self.compile_command(build_target=build_target, **env_vars)
+ cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
+ if not dry_run:
+ cli.echo('\n')
+ ret = cli.run(command, capture_output=False)
+ if ret.returncode:
+ return ret.returncode
+
+
+class KeyboardKeymapBuildTarget(BuildTarget):
+ def __init__(self, keyboard: str, keymap: str, json: dict = None):
+ super().__init__(keyboard=keyboard, keymap=keymap, json=json)
+
+ def __repr__(self):
+ return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})'
+
+ def _load_json(self):
+ self._json = keymap_json(self.keyboard, self.keymap)
+
+ def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+ pass
+
+ def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+ compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
+
+ for key, value in env_vars.items():
+ compile_args.append(f'{key}={value}')
+
+ return compile_args
+
+
+class JsonKeymapBuildTarget(BuildTarget):
+ def __init__(self, json_path):
+ if isinstance(json_path, Path):
+ self.json_path = json_path
+ else:
+ self.json_path = None
+
+ json = parse_configurator_json(json_path) # Will load from stdin if provided
+
+ # In case the user passes a keymap.json from a keymap directory directly to the CLI.
+ # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
+ json["keymap"] = json.get("keymap", "default_json")
+
+ super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json)
+
+ self._keymap_json = self._generated_files_path / 'keymap.json'
+
+ def __repr__(self):
+ return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
+
+ def _load_json(self):
+ pass # Already loaded in constructor
+
+ def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+ if self._clean:
+ if self._intermediate_output.exists():
+ shutil.rmtree(self._intermediate_output)
+
+ # begin with making the deepest folder in the tree
+ self._generated_files_path.mkdir(exist_ok=True, parents=True)
+
+ # Compare minified to ensure consistent comparison
+ new_content = json.dumps(self.json, separators=(',', ':'))
+ if self._keymap_json.exists():
+ old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
+ if old_content == new_content:
+ new_content = None
+
+ # Write the keymap.json file if different so timestamps are only updated
+ # if the content changes -- running `make` won't treat it as modified.
+ if new_content:
+ self._keymap_json.write_text(new_content, encoding='utf-8')
+
+ def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+ compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
+ compile_args.extend([
+ f'MAIN_KEYMAP_PATH_1={self._intermediate_output}',
+ f'MAIN_KEYMAP_PATH_2={self._intermediate_output}',
+ f'MAIN_KEYMAP_PATH_3={self._intermediate_output}',
+ f'MAIN_KEYMAP_PATH_4={self._intermediate_output}',
+ f'MAIN_KEYMAP_PATH_5={self._intermediate_output}',
+ f'KEYMAP_JSON={self._keymap_json}',
+ f'KEYMAP_PATH={self._generated_files_path}',
+ ])
+
+ for key, value in env_vars.items():
+ compile_args.append(f'{key}={value}')
+
+ return compile_args
M lib/python/qmk/cli/clean.py => lib/python/qmk/cli/clean.py +2 -2
@@ 2,7 2,7 @@
"""
from subprocess import DEVNULL
-from qmk.commands import create_make_target
+from qmk.commands import find_make
from milc import cli
@@ 11,4 11,4 @@ from milc import cli
def clean(cli):
"""Runs `make clean` (or `make distclean` if --all is passed)
"""
- cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
+ cli.run([find_make(), 'distclean' if cli.args.all else 'clean'], capture_output=False, stdin=DEVNULL)
M lib/python/qmk/cli/compile.py => lib/python/qmk/cli/compile.py +14 -44
@@ 7,22 7,11 @@ from argcomplete.completers import FilesCompleter
from milc import cli
import qmk.path
-from qmk.constants import QMK_FIRMWARE
from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
+from qmk.commands import build_environment
from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards
from qmk.keymap import keymap_completer, locate_keymap
-from qmk.cli.generate.compilation_database import write_compilation_database
-
-
-def _is_keymap_target(keyboard, keymap):
- if keymap == 'all':
- return True
-
- if locate_keymap(keyboard, keymap):
- return True
-
- return False
+from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
@@ 32,6 21,7 @@ def _is_keymap_target(keyboard, keymap):
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
+@cli.argument('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.")
@cli.argument('--compiledb', arg_only=True, action='store_true', help="Generates the clang compile_commands.json file during build. Implies --clean.")
@cli.subcommand('Compile a QMK Firmware.')
@automagic_keyboard
@@ 53,47 43,27 @@ def compile(cli):
# Build the environment vars
envs = build_environment(cli.args.env)
- # Determine the compile command
- commands = []
-
- current_keyboard = None
- current_keymap = None
+ # Handler for the build target
+ target = None
if cli.args.filename:
- # If a configurator JSON was provided generate a keymap and compile it
- user_keymap = parse_configurator_json(cli.args.filename)
- commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
+ # if we were given a filename, assume we have a json build target
+ target = JsonKeymapBuildTarget(cli.args.filename)
elif cli.config.compile.keyboard and cli.config.compile.keymap:
- # Generate the make command for a specific keyboard/keymap.
- if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
+ # if we got a keyboard and keymap, attempt to find it
+ if not locate_keymap(cli.config.compile.keyboard, cli.config.compile.keymap):
cli.log.error('Invalid keymap argument.')
cli.print_help()
return False
- if cli.args.clean:
- commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
- commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
+ # If we got here, then we have a valid keyboard and keymap for a build target
+ target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap)
- current_keyboard = cli.config.compile.keyboard
- current_keymap = cli.config.compile.keymap
-
- if not commands:
+ if not target:
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
cli.print_help()
return False
- if cli.args.compiledb:
- if current_keyboard is None or current_keymap is None:
- cli.log.error('You must supply both `--keyboard` and `--keymap` or be in a directory with a keymap to generate a compile_commands.json file.')
- cli.print_help()
- return False
- write_compilation_database(current_keyboard, current_keymap, QMK_FIRMWARE / 'compile_commands.json')
-
- cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
- if not cli.args.dry_run:
- cli.echo('\n')
- for command in commands:
- ret = cli.run(command, capture_output=False)
- if ret.returncode:
- return ret.returncode
+ target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb)
+ target.compile(cli.args.target, dry_run=cli.args.dry_run, **envs)
M lib/python/qmk/cli/find.py => lib/python/qmk/cli/find.py +5 -9
@@ 19,13 19,9 @@ from qmk.search import search_keymap_targets
def find(cli):
"""Search through all keyboards and keymaps for a given search criteria.
"""
+ targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter)
+ for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
+ print(f'{target}')
- if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
- cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
-
- targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print)
- for keyboard, keymap, print_vals in targets:
- print(f'{keyboard}:{keymap}')
-
- for key, val in print_vals:
- print(f' {key}={val}')
+ for key in cli.args.print:
+ print(f' {key}={target.dotty.get(key, None)}')
M lib/python/qmk/cli/flash.py => lib/python/qmk/cli/flash.py +15 -30
@@ 4,25 4,17 @@ You can compile a keymap already in the repo or using a QMK Configurator export.
A bootloader must be specified.
"""
from argcomplete.completers import FilesCompleter
+from pathlib import Path
from milc import cli
import qmk.path
from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
+from qmk.commands import build_environment
from qmk.keyboard import keyboard_completer, keyboard_folder
from qmk.keymap import keymap_completer, locate_keymap
from qmk.flashers import flasher
-
-
-def _is_keymap_target(keyboard, keymap):
- if keymap == 'all':
- return True
-
- if locate_keymap(keyboard, keymap):
- return True
-
- return False
+from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
def _list_bootloaders():
@@ 89,7 81,7 @@ def flash(cli):
If bootloader is omitted the make system will use the configured bootloader for that keyboard.
"""
- if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
+ if cli.args.filename and isinstance(cli.args.filename, Path) and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
return _flash_binary(cli.args.filename, cli.args.mcu)
if cli.args.bootloaders:
@@ 98,34 90,27 @@ def flash(cli):
# Build the environment vars
envs = build_environment(cli.args.env)
- # Determine the compile command
- commands = []
+ # Handler for the build target
+ target = None
if cli.args.filename:
- # If a configurator JSON was provided generate a keymap and compile it
- user_keymap = parse_configurator_json(cli.args.filename)
- commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)]
+ # if we were given a filename, assume we have a json build target
+ target = JsonKeymapBuildTarget(cli.args.filename)
elif cli.config.flash.keyboard and cli.config.flash.keymap:
- # Generate the make command for a specific keyboard/keymap.
- if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
+ # if we got a keyboard and keymap, attempt to find it
+ if not locate_keymap(cli.config.flash.keyboard, cli.config.flash.keymap):
cli.log.error('Invalid keymap argument.')
cli.print_help()
return False
- if cli.args.clean:
- commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))
- commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs))
+ # If we got here, then we have a valid keyboard and keymap for a build target
+ target = KeyboardKeymapBuildTarget(cli.config.flash.keyboard, cli.config.flash.keymap)
- if not commands:
+ if not target:
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
cli.print_help()
return False
- cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
- if not cli.args.dry_run:
- cli.echo('\n')
- for command in commands:
- ret = cli.run(command, capture_output=False)
- if ret.returncode:
- return ret.returncode
+ target.configure(parallel=cli.config.flash.parallel, clean=cli.args.clean)
+ target.compile(cli.args.bootloader, dry_run=cli.args.dry_run, **envs)
M lib/python/qmk/cli/generate/compilation_database.py => lib/python/qmk/cli/generate/compilation_database.py +10 -6
@@ 12,7 12,7 @@ from typing import Dict, Iterator, List, Union
from milc import cli, MILC
-from qmk.commands import create_make_command
+from qmk.commands import find_make
from qmk.constants import QMK_FIRMWARE
from qmk.decorators import automagic_keyboard, automagic_keymap
from qmk.keyboard import keyboard_completer, keyboard_folder
@@ 76,9 76,12 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
return records
-def write_compilation_database(keyboard: str, keymap: str, output_path: Path) -> bool:
+def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool:
# Generate the make command for a specific keyboard/keymap.
- command = create_make_command(keyboard, keymap, dry_run=True)
+ if not command:
+ from qmk.build_targets import KeyboardKeymapBuildTarget # Lazy load due to circular references
+ target = KeyboardKeymapBuildTarget(keyboard, keymap)
+ command = target.compile_command(dry_run=True, **env_vars)
if not command:
cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
@@ 90,9 93,10 @@ def write_compilation_database(keyboard: str, keymap: str, output_path: Path) ->
env.pop("MAKEFLAGS", None)
# re-use same executable as the main make invocation (might be gmake)
- clean_command = [command[0], 'clean']
- cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
- cli.run(clean_command, capture_output=False, check=True, env=env)
+ if not skip_clean:
+ clean_command = [find_make(), "clean"]
+ cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
+ cli.run(clean_command, capture_output=False, check=True, env=env)
cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command))
M lib/python/qmk/cli/mass_compile.py => lib/python/qmk/cli/mass_compile.py +19 -10
@@ 3,26 3,28 @@
This will compile everything in parallel, for testing purposes.
"""
import os
+from typing import List
from pathlib import Path
from subprocess import DEVNULL
from milc import cli
from qmk.constants import QMK_FIRMWARE
-from qmk.commands import _find_make, get_make_parallel_args
+from qmk.commands import find_make, get_make_parallel_args, build_environment
from qmk.search import search_keymap_targets, search_make_targets
+from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget
-def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
+def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env):
if len(targets) == 0:
return
- make_cmd = _find_make()
+ make_cmd = find_make()
builddir = Path(QMK_FIRMWARE) / '.build'
makefile = builddir / 'parallel_kb_builds.mk'
if dry_run:
cli.log.info('Compilation targets:')
- for target in sorted(targets):
+ for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
cli.log.info(f"{{fg_cyan}}qmk compile -kb {target[0]} -km {target[1]}{{fg_reset}}")
else:
if clean:
@@ 30,9 32,13 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
builddir.mkdir(parents=True, exist_ok=True)
with open(makefile, "w") as f:
- for target in sorted(targets):
- keyboard_name = target[0]
- keymap_name = target[1]
+ for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
+ keyboard_name = target.keyboard
+ keymap_name = target.keymap
+ target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation
+ target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
+ command = target.compile_command(**env)
+ command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism
keyboard_safe = keyboard_name.replace('/', '_')
build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
@@ 43,7 49,7 @@ all: {keyboard_safe}_{keymap_name}_binary
{keyboard_safe}_{keymap_name}_binary:
@rm -f "{build_log}" || true
@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
- +@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(env)} \\
+ {' '.join(command)} \\
>>"{build_log}" 2>&1 \\
|| cp "{build_log}" "{failed_log}"
@{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
@@ 95,8 101,11 @@ def mass_compile(cli):
"""Compile QMK Firmware against all keyboards.
"""
if len(cli.args.builds) > 0:
- targets = search_make_targets(cli.args.builds, cli.args.filter)
+ json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
+ make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
+ targets = search_make_targets(make_like_targets)
+ targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets])
else:
targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
- return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env)
+ return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, **build_environment(cli.args.env))
M lib/python/qmk/commands.py => lib/python/qmk/commands.py +2 -162
@@ 2,18 2,16 @@
"""
import os
import sys
-import json
import shutil
from pathlib import Path
from milc import cli
import jsonschema
-from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX
from qmk.json_schema import json_load, validate
-def _find_make():
+def find_make():
"""Returns the correct make command for this environment.
"""
make_cmd = os.environ.get('MAKE')
@@ 24,74 22,6 @@ def _find_make():
return make_cmd
-def create_make_target(target, dry_run=False, parallel=1, **env_vars):
- """Create a make command
-
- Args:
-
- target
- Usually a make rule, such as 'clean' or 'all'.
-
- dry_run
- make -n -- don't actually build
-
- parallel
- The number of make jobs to run in parallel
-
- **env_vars
- Environment variables to be passed to make.
-
- Returns:
-
- A command that can be run to make the specified keyboard and keymap
- """
- env = []
- make_cmd = _find_make()
-
- for key, value in env_vars.items():
- env.append(f'{key}={value}')
-
- if cli.config.general.verbose:
- env.append('VERBOSE=true')
-
- return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target]
-
-
-def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars):
- """Create a make compile command
-
- Args:
-
- keyboard
- The path of the keyboard, for example 'plank'
-
- keymap
- The name of the keymap, for example 'algernon'
-
- target
- Usually a bootloader.
-
- dry_run
- make -n -- don't actually build
-
- parallel
- The number of make jobs to run in parallel
-
- **env_vars
- Environment variables to be passed to make.
-
- Returns:
-
- A command that can be run to make the specified keyboard and keymap
- """
- make_args = [keyboard, keymap]
-
- if target:
- make_args.append(target)
-
- return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars)
-
-
def get_make_parallel_args(parallel=1):
"""Returns the arguments for running the specified number of parallel jobs.
"""
@@ 100,7 30,7 @@ def get_make_parallel_args(parallel=1):
if int(parallel) <= 0:
# 0 or -1 means -j without argument (unlimited jobs)
parallel_args.append('--jobs')
- else:
+ elif int(parallel) > 1:
parallel_args.append('--jobs=' + str(parallel))
if int(parallel) != 1:
@@ 110,96 40,6 @@ def get_make_parallel_args(parallel=1):
return parallel_args
-def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars):
- """Convert a configurator export JSON file into a C file and then compile it.
-
- Args:
-
- user_keymap
- A deserialized keymap export
-
- bootloader
- A bootloader to flash
-
- parallel
- The number of make jobs to run in parallel
-
- Returns:
-
- A command to run to compile and flash the C file.
- """
- # In case the user passes a keymap.json from a keymap directory directly to the CLI.
- # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
- user_keymap["keymap"] = user_keymap.get("keymap", "default_json")
-
- keyboard_filesafe = user_keymap['keyboard'].replace('/', '_')
- target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
- intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{keyboard_filesafe}_{user_keymap["keymap"]}')
- keymap_dir = intermediate_output / 'src'
- keymap_json = keymap_dir / 'keymap.json'
-
- if clean:
- if intermediate_output.exists():
- shutil.rmtree(intermediate_output)
-
- # begin with making the deepest folder in the tree
- keymap_dir.mkdir(exist_ok=True, parents=True)
-
- # Compare minified to ensure consistent comparison
- new_content = json.dumps(user_keymap, separators=(',', ':'))
- if keymap_json.exists():
- old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
- if old_content == new_content:
- new_content = None
-
- # Write the keymap.json file if different
- if new_content:
- keymap_json.write_text(new_content, encoding='utf-8')
-
- # Return a command that can be run to make the keymap and flash if given
- verbose = 'true' if cli.config.general.verbose else 'false'
- color = 'true' if cli.config.general.color else 'false'
- make_command = [_find_make()]
-
- if not cli.config.general.verbose:
- make_command.append('-s')
-
- make_command.extend([
- *get_make_parallel_args(parallel),
- '-r',
- '-R',
- '-f',
- 'builddefs/build_keyboard.mk',
- ])
-
- if bootloader:
- make_command.append(bootloader)
-
- make_command.extend([
- f'KEYBOARD={user_keymap["keyboard"]}',
- f'KEYMAP={user_keymap["keymap"]}',
- f'KEYBOARD_FILESAFE={keyboard_filesafe}',
- f'TARGET={target}',
- f'INTERMEDIATE_OUTPUT={intermediate_output}',
- f'MAIN_KEYMAP_PATH_1={intermediate_output}',
- f'MAIN_KEYMAP_PATH_2={intermediate_output}',
- f'MAIN_KEYMAP_PATH_3={intermediate_output}',
- f'MAIN_KEYMAP_PATH_4={intermediate_output}',
- f'MAIN_KEYMAP_PATH_5={intermediate_output}',
- f'KEYMAP_JSON={keymap_json}',
- f'KEYMAP_PATH={keymap_dir}',
- f'VERBOSE={verbose}',
- f'COLOR={color}',
- 'SILENT=false',
- 'QMK_BIN="qmk"',
- ])
-
- for key, value in env_vars.items():
- make_command.append(f'{key}={value}')
-
- return make_command
-
-
def parse_configurator_json(configurator_file):
"""Open and parse a configurator json export
"""
M lib/python/qmk/keyboard.py => lib/python/qmk/keyboard.py +1 -5
@@ 98,11 98,7 @@ def keyboard_folder(keyboard):
if keyboard == last_keyboard:
break
- rules_mk_file = Path(base_path, keyboard, 'rules.mk')
-
- if rules_mk_file.exists():
- rules_mk = parse_rules_mk_file(rules_mk_file)
- keyboard = rules_mk.get('DEFAULT_FOLDER', keyboard)
+ keyboard = resolve_keyboard(keyboard)
if not qmk.path.is_keyboard(keyboard):
raise ValueError(f'Invalid keyboard: {keyboard}')
M lib/python/qmk/search.py => lib/python/qmk/search.py +17 -17
@@ 11,8 11,9 @@ from milc import cli
from qmk.util import parallel_map
from qmk.info import keymap_json
-import qmk.keyboard
-import qmk.keymap
+from qmk.keyboard import list_keyboards, keyboard_folder
+from qmk.keymap import list_keymaps, locate_keymap
+from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget
def _set_log_level(level):
@@ 36,15 37,15 @@ def _all_keymaps(keyboard):
"""Returns a list of tuples of (keyboard, keymap) for all keymaps for the given keyboard.
"""
with ignore_logging():
- keyboard = qmk.keyboard.resolve_keyboard(keyboard)
- return [(keyboard, keymap) for keymap in qmk.keymap.list_keymaps(keyboard)]
+ keyboard = keyboard_folder(keyboard)
+ return [(keyboard, keymap) for keymap in list_keymaps(keyboard)]
def _keymap_exists(keyboard, keymap):
"""Returns the keyboard name if the keyboard+keymap combination exists, otherwise None.
"""
with ignore_logging():
- return keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
+ return keyboard if locate_keymap(keyboard, keymap) is not None else None
def _load_keymap_info(kb_km):
@@ 75,7 76,7 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
Caters for 'all' in either keyboard or keymap, or both.
"""
if all_keyboards is None:
- all_keyboards = qmk.keyboard.list_keyboards()
+ all_keyboards = list_keyboards()
if keyboard == 'all':
if keymap == 'all':
@@ 90,30 91,29 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
else:
if keymap == 'all':
- keyboard = qmk.keyboard.resolve_keyboard(keyboard)
cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...')
return _all_keymaps(keyboard)
else:
- return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)]
+ return [(keyboard, keymap)]
def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
"""Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.
"""
overall_targets = []
- all_keyboards = qmk.keyboard.list_keyboards()
+ all_keyboards = list_keyboards()
for target in targets:
overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))
return list(sorted(set(overall_targets)))
-def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
"""Filter a list of (keyboard, keymap) tuples based on the supplied filters.
Optionally includes the values of the queried info.json keys.
"""
- if len(filters) == 0 and len(print_vals) == 0:
- targets = [(kb, km, {}) for kb, km in target_list]
+ if len(filters) == 0:
+ targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list]
else:
cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
@@ 172,18 172,18 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
continue
- targets = [(e[0], e[1], [(p, e[2].get(p)) for p in print_vals]) for e in valid_keymaps]
+ targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps]
return targets
-def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:
"""Search for build targets matching the supplied criteria.
"""
- return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
+ return _filter_keymap_targets(expand_keymap_targets(targets), filters)
-def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_make_targets(targets: List[str], filters: List[str] = []) -> List[BuildTarget]:
"""Search for build targets matching the supplied criteria.
"""
- return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
+ return _filter_keymap_targets(expand_make_targets(targets), filters)