# SPDX-License-Identifier: Apache-2.0 # Copyright 2013-2021 The Meson development team from __future__ import annotations from .base import ExternalDependency, DependencyException, DependencyTypeName from .pkgconfig import PkgConfigDependency from ..mesonlib import (Popen_safe, join_args, version_compare) from ..options import OptionKey from ..programs import ExternalProgram from .. import mlog import re import os import json import typing as T if T.TYPE_CHECKING: from typing_extensions import TypedDict from ..environment import Environment # Definition of what `dub describe` returns (only the fields used by Meson) class DubDescription(TypedDict): platform: T.List[str] architecture: T.List[str] buildType: str packages: T.List[DubPackDesc] targets: T.List[DubTargetDesc] class DubPackDesc(TypedDict): name: str version: str active: bool configuration: str path: str targetType: str targetFileName: str class DubTargetDesc(TypedDict): rootPackage: str linkDependencies: T.List[str] buildSettings: DubBuildSettings cacheArtifactPath: str class DubBuildSettings(TypedDict): importPaths: T.List[str] stringImportPaths: T.List[str] versions: T.List[str] mainSourceFile: str sourceFiles: T.List[str] dflags: T.List[str] libs: T.List[str] lflags: T.List[str] class FindTargetEntry(TypedDict): search: str artifactPath: str class DubDependency(ExternalDependency): # dub program and version class_dubbin: T.Optional[T.Tuple[ExternalProgram, str]] = None class_dubbin_searched = False class_cache_dir = '' # Map Meson Compiler ID's to Dub Compiler ID's _ID_MAP: T.Mapping[str, str] = { 'dmd': 'dmd', 'gcc': 'gdc', 'llvm': 'ldc', } def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]): super().__init__(DependencyTypeName('dub'), environment, kwargs, language='d') self.name = name from ..compilers.d import DCompiler, d_feature_args _temp_comp = super().get_compiler() assert isinstance(_temp_comp, DCompiler) self.compiler = _temp_comp if 'required' in kwargs: self.required = kwargs.get('required') if DubDependency.class_dubbin is None and not DubDependency.class_dubbin_searched: DubDependency.class_dubbin = self._check_dub() DubDependency.class_dubbin_searched = True if DubDependency.class_dubbin is None: if self.required: raise DependencyException('DUB not found.') self.is_found = False return (self.dubbin, dubver) = DubDependency.class_dubbin # pylint: disable=unpacking-non-sequence assert isinstance(self.dubbin, ExternalProgram) # Check if Dub's compatibility with Meson self._search_in_cache = version_compare(dubver, '<=1.31.1') self._use_cache_describe = version_compare(dubver, '>=1.35.0') self._dub_has_build_deep = version_compare(dubver, '>=1.35.0') if not self._search_in_cache and not self._use_cache_describe: if self.required: raise DependencyException( f'DUB version {dubver} is not compatible with Meson' " (can't locate artifacts in DUB's cache). Upgrade to Dub >= 1.35.") else: mlog.warning(f'DUB dependency {name} not found because Dub {dubver} ' "is not compatible with Meson. (Can't locate artifacts in DUB's cache)." ' Upgrade to Dub >= 1.35') self.is_found = False return mlog.debug('Determining dependency {!r} with DUB executable ' '{!r}'.format(name, self.dubbin.get_path())) # if an explicit version spec was stated, use this when querying Dub main_pack_spec = name if 'version' in kwargs: version_spec = kwargs['version'] if isinstance(version_spec, list): version_spec = " ".join(version_spec) main_pack_spec = f'{name}@{version_spec}' # we need to know the target architecture dub_arch = self.compiler.arch # we need to know the build type as well dub_buildtype = str(environment.coredata.get_option(OptionKey('buildtype'))) # MESON types: choices=['plain', 'debug', 'debugoptimized', 'release', 'minsize', 'custom'])), # DUB types: debug (default), plain, release, release-debug, release-nobounds, unittest, profile, profile-gc, # docs, ddox, cov, unittest-cov, syntax and custom if dub_buildtype == 'debugoptimized': dub_buildtype = 'release-debug' elif dub_buildtype == 'minsize': dub_buildtype = 'release' # A command that might be useful in case of missing DUB package def dub_build_deep_command() -> str: if self._dub_has_build_deep: cmd = ['dub', 'build', '--deep'] else: cmd = ['dub', 'run', '--yes', 'dub-build-deep', '--'] return join_args(cmd + [ main_pack_spec, '--arch=' + dub_arch, '--compiler=' + self.compiler.get_exelist()[-1], '--build=' + dub_buildtype ]) # Ask dub for the package describe_cmd = [ 'describe', main_pack_spec, '--arch=' + dub_arch, '--build=' + dub_buildtype, '--compiler=' + self.compiler.get_exelist()[-1] ] ret, res, err = self._call_dubbin(describe_cmd) if ret != 0: mlog.debug('DUB describe failed: ' + err) if 'locally' in err: mlog.error(mlog.bold(main_pack_spec), 'is not present locally. You may try the following command:') mlog.log(mlog.bold(dub_build_deep_command())) self.is_found = False return dub_comp_id = self._ID_MAP[self.compiler.get_id()] description: DubDescription = json.loads(res) self.compile_args = [] self.link_args = self.raw_link_args = [] show_buildtype_warning = False # collect all targets targets = {t['rootPackage']: t for t in description['targets']} def find_package_target(pkg: DubPackDesc) -> bool: nonlocal show_buildtype_warning # try to find a static library in a DUB folder corresponding to # version, configuration, compiler, arch and build-type # if can find, add to link_args. # link_args order is meaningful, so this function MUST be called in the right order pack_id = f'{pkg["name"]}@{pkg["version"]}' tgt_desc = targets[pkg['name']] (tgt_file, compatibilities) = self._find_target_in_cache(description, pkg, tgt_desc, dub_comp_id) if tgt_file is None: if not compatibilities: mlog.error(mlog.bold(pack_id), 'not found') elif 'compiler' not in compatibilities: mlog.error(mlog.bold(pack_id), 'found but not compiled with ', mlog.bold(dub_comp_id)) elif dub_comp_id != 'gdc' and 'compiler_version' not in compatibilities: mlog.error(mlog.bold(pack_id), 'found but not compiled with', mlog.bold(f'{dub_comp_id}-{self.compiler.version}')) elif 'arch' not in compatibilities: mlog.error(mlog.bold(pack_id), 'found but not compiled for', mlog.bold(dub_arch)) elif 'platform' not in compatibilities: mlog.error(mlog.bold(pack_id), 'found but not compiled for', mlog.bold('.'.join(description['platform']))) elif 'configuration' not in compatibilities: mlog.error(mlog.bold(pack_id), 'found but not compiled for the', mlog.bold(pkg['configuration']), 'configuration') else: mlog.error(mlog.bold(pack_id), 'not found') mlog.log('You may try the following command to install the necessary DUB libraries:') mlog.log(mlog.bold(dub_build_deep_command())) return False if 'build_type' not in compatibilities: mlog.warning(mlog.bold(pack_id), 'found but not compiled as', mlog.bold(dub_buildtype)) show_buildtype_warning = True self.link_args.append(tgt_file) return True # Main algorithm: # 1. Ensure that the target is a compatible library type (not dynamic) # 2. Find a compatible built library for the main dependency # 3. Do the same for each sub-dependency. # link_args MUST be in the same order than the "linkDependencies" of the main target # 4. Add other build settings (imports, versions etc.) # 1 self.is_found = False packages: T.Dict[str, DubPackDesc] = {} for pkg in description['packages']: packages[pkg['name']] = pkg if not pkg['active']: continue if pkg['targetType'] == 'dynamicLibrary': mlog.error('DUB dynamic library dependencies are not supported.') self.is_found = False return # check that the main dependency is indeed a library if pkg['name'] == name: self.is_found = True if pkg['targetType'] not in ['library', 'sourceLibrary', 'staticLibrary']: mlog.error(mlog.bold(name), "found but it isn't a library") self.is_found = False return self.version = pkg['version'] self.pkg = pkg if name not in targets: self.is_found = False if self.pkg['targetType'] == 'sourceLibrary': # source libraries have no associated targets, # but some build settings like import folders must be found from the package object. # Current algo only get these from "buildSettings" in the target object. # Let's save this for a future PR. # (See openssl DUB package for example of sourceLibrary) mlog.error('DUB targets of type', mlog.bold('sourceLibrary'), 'are not supported.') else: mlog.error('Could not find target description for', mlog.bold(main_pack_spec)) if not self.is_found: mlog.error(f'Could not find {name} in DUB description') return # Current impl only supports static libraries self.static = True # 2 if not find_package_target(self.pkg): self.is_found = False return # 3 for link_dep in targets[name]['linkDependencies']: pkg = packages[link_dep] if not find_package_target(pkg): self.is_found = False return if show_buildtype_warning: mlog.log('If it is not suitable, try the following command and reconfigure Meson with', mlog.bold('--clearcache')) mlog.log(mlog.bold(dub_build_deep_command())) # 4 bs = targets[name]['buildSettings'] for flag in bs['dflags']: self.compile_args.append(flag) for path in bs['importPaths']: self.compile_args.append('-I' + path) for path in bs['stringImportPaths']: if 'import_dir' not in d_feature_args[self.compiler.id]: break flag = d_feature_args[self.compiler.id]['import_dir'] self.compile_args.append(f'{flag}={path}') for ver in bs['versions']: if 'version' not in d_feature_args[self.compiler.id]: break flag = d_feature_args[self.compiler.id]['version'] self.compile_args.append(f'{flag}={ver}') if bs['mainSourceFile']: self.compile_args.append(bs['mainSourceFile']) # pass static libraries # linkerFiles are added during step 3 # for file in bs['linkerFiles']: # self.link_args.append(file) for file in bs['sourceFiles']: # sourceFiles may contain static libraries if file.endswith('.lib') or file.endswith('.a'): self.link_args.append(file) for flag in bs['lflags']: self.link_args.append(flag) is_windows = self.env.machines.host.is_windows() if is_windows: winlibs = ['kernel32', 'user32', 'gdi32', 'winspool', 'shell32', 'ole32', 'oleaut32', 'uuid', 'comdlg32', 'advapi32', 'ws2_32'] for lib in bs['libs']: if os.name != 'nt': # trying to add system libraries by pkg-config pkgdep = PkgConfigDependency(lib, environment, {'required': 'true', 'silent': 'true'}) if pkgdep.is_found: for arg in pkgdep.get_compile_args(): self.compile_args.append(arg) for arg in pkgdep.get_link_args(): self.link_args.append(arg) for arg in pkgdep.get_link_args(raw=True): self.raw_link_args.append(arg) continue if is_windows and lib in winlibs: self.link_args.append(lib + '.lib') continue # fallback self.link_args.append('-l'+lib) # This function finds the target of the provided JSON package, built for the right # compiler, architecture, configuration... # It returns (target|None, {compatibilities}) # If None is returned for target, compatibilities will list what other targets were found without full compatibility def _find_target_in_cache(self, desc: DubDescription, pkg_desc: DubPackDesc, tgt_desc: DubTargetDesc, dub_comp_id: str ) -> T.Tuple[T.Optional[str], T.Set[str]]: mlog.debug('Searching in DUB cache for compatible', pkg_desc['targetFileName']) # recent DUB versions include a direct path to a compatible cached artifact if self._use_cache_describe: tgt_file = tgt_desc['cacheArtifactPath'] if os.path.exists(tgt_file): return (tgt_file, {'configuration', 'platform', 'arch', 'compiler', 'compiler_version', 'build_type'}) else: return (None, set()) assert self._search_in_cache # try to find a string like library-debug-linux.posix-x86_64-ldc_2081-EF934983A3319F8F8FF2F0E107A363BA # fields are: # - configuration # - build type # - platform # - architecture # - compiler id (dmd, ldc, gdc) # - compiler version or frontend id or frontend version? comp_versions = self._get_comp_versions_to_find(dub_comp_id) # build_type is not in check_list because different build types might be compatible. # We do show a WARNING that the build type is not the same. # It might be critical in release builds, and acceptable otherwise check_list = {'configuration', 'platform', 'arch', 'compiler', 'compiler_version'} compatibilities: T.Set[str] = set() for entry in self._cache_entries(pkg_desc): target = entry['artifactPath'] if not os.path.exists(target): # unless Dub and Meson are racing, the target file should be present # when the directory is present mlog.debug("WARNING: Could not find a Dub target: " + target) continue # we build a new set for each entry, because if this target is returned # we want to return only the compatibilities associated to this target # otherwise we could miss the WARNING about build_type comps: T.Set[str] = set() search = entry['search'] mlog.debug('searching compatibility in ' + search) mlog.debug('compiler_versions', comp_versions) if pkg_desc['configuration'] in search: comps.add('configuration') if desc['buildType'] in search: comps.add('build_type') if all(platform in search for platform in desc['platform']): comps.add('platform') if all(arch in search for arch in desc['architecture']): comps.add('arch') if dub_comp_id in search: comps.add('compiler') if not comp_versions or any(cv in search for cv in comp_versions): comps.add('compiler_version') if check_list.issubset(comps): mlog.debug('Found', target) return (target, comps) else: compatibilities = set.union(compatibilities, comps) return (None, compatibilities) def _cache_entries(self, pkg_desc: DubPackDesc) -> T.List[FindTargetEntry]: # the "old" cache is the `.dub` directory in every package of ~/.dub/packages dub_build_path = os.path.join(pkg_desc['path'], '.dub', 'build') if not os.path.exists(dub_build_path): mlog.warning('No such cache folder:', dub_build_path) return [] mlog.debug('Checking in DUB cache folder', dub_build_path) return [ { 'search': dir_entry, 'artifactPath': os.path.join(dub_build_path, dir_entry, pkg_desc['targetFileName']) } for dir_entry in os.listdir(dub_build_path) ] def _get_comp_versions_to_find(self, dub_comp_id: str) -> T.List[str]: # Get D frontend version implemented in the compiler, or the compiler version itself # gdc doesn't support this if dub_comp_id == 'gdc': return [] comp_versions = [self.compiler.version] ret, res = self._call_compbin(['--version'])[0:2] if ret != 0: mlog.error('Failed to run', mlog.bold(' '.join(self.dubbin.get_command() + ['--version']))) return [] d_ver_reg = re.search('v[0-9].[0-9][0-9][0-9].[0-9]', res) # Ex.: v2.081.2 if d_ver_reg is not None: frontend_version = d_ver_reg.group() frontend_id = frontend_version.rsplit('.', 1)[0].replace( 'v', '').replace('.', '') # Fix structure. Ex.: 2081 comp_versions.extend([frontend_version, frontend_id]) return comp_versions def _call_dubbin(self, args: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> T.Tuple[int, str, str]: assert isinstance(self.dubbin, ExternalProgram) p, out, err = Popen_safe(self.dubbin.get_command() + args, env=env) return p.returncode, out.strip(), err.strip() def _call_compbin(self, args: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> T.Tuple[int, str, str]: p, out, err = Popen_safe(self.compiler.get_exelist() + args, env=env) return p.returncode, out.strip(), err.strip() def _check_dub(self) -> T.Optional[T.Tuple[ExternalProgram, str]]: def find() -> T.Optional[T.Tuple[ExternalProgram, str]]: dubbin = ExternalProgram('dub', silent=True) if not dubbin.found(): return None try: p, out = Popen_safe(dubbin.get_command() + ['--version'])[0:2] if p.returncode != 0: mlog.warning('Found dub {!r} but couldn\'t run it' ''.format(' '.join(dubbin.get_command()))) return None except (FileNotFoundError, PermissionError): return None vermatch = re.search(r'DUB version (\d+\.\d+\.\d+.*), ', out.strip()) if vermatch: dubver = vermatch.group(1) else: mlog.warning(f"Found dub {' '.join(dubbin.get_command())} but couldn't parse version in {out.strip()}") return None return (dubbin, dubver) found = find() if found is None: mlog.log('Found DUB:', mlog.red('NO')) else: (dubbin, dubver) = found mlog.log('Found DUB:', mlog.bold(dubbin.get_path()), '(version %s)' % dubver) return found