import errno import fnmatch import re from collections import namedtuple, OrderedDict from functools import wraps from typing import Optional, Dict, Any, List, Union, Callable, Iterable import six import yaml from ceph.deployment.hostspec import HostSpec from ceph.deployment.utils import unwrap_ipv6 class ServiceSpecValidationError(Exception): """ Defining an exception here is a bit problematic, cause you cannot properly catch it, if it was raised in a different mgr module. """ def __init__(self, msg: str, errno: int = -errno.EINVAL): super(ServiceSpecValidationError, self).__init__(msg) self.errno = errno def assert_valid_host(name): p = re.compile('^[a-zA-Z0-9-]+$') try: assert len(name) <= 250, 'name is too long (max 250 chars)' for part in name.split('.'): assert len(part) > 0, '.-delimited name component must not be empty' assert len(part) <= 63, '.-delimited name component must not be more than 63 chars' assert p.match(part), 'name component must include only a-z, 0-9, and -' except AssertionError as e: raise ServiceSpecValidationError(e) def handle_type_error(method): @wraps(method) def inner(cls, *args, **kwargs): try: return method(cls, *args, **kwargs) except (TypeError, AttributeError) as e: error_msg = '{}: {}'.format(cls.__name__, e) raise ServiceSpecValidationError(error_msg) return inner class HostPlacementSpec(namedtuple('HostPlacementSpec', ['hostname', 'network', 'name'])): def __str__(self): res = '' res += self.hostname if self.network: res += ':' + self.network if self.name: res += '=' + self.name return res @classmethod @handle_type_error def from_json(cls, data): if isinstance(data, str): return cls.parse(data) return cls(**data) def to_json(self) -> str: return str(self) @classmethod def parse(cls, host, require_network=True): # type: (str, bool) -> HostPlacementSpec """ Split host into host, network, and (optional) daemon name parts. The network part can be an IP, CIDR, or ceph addrvec like '[v2:1.2.3.4:3300,v1:1.2.3.4:6789]'. e.g., "myhost" "myhost=name" "myhost:1.2.3.4" "myhost:1.2.3.4=name" "myhost:1.2.3.0/24" "myhost:1.2.3.0/24=name" "myhost:[v2:1.2.3.4:3000]=name" "myhost:[v2:1.2.3.4:3000,v1:1.2.3.4:6789]=name" """ # Matches from start to : or = or until end of string host_re = r'^(.*?)(:|=|$)' # Matches from : to = or until end of string ip_re = r':(.*?)(=|$)' # Matches from = to end of string name_re = r'=(.*?)$' # assign defaults host_spec = cls('', '', '') match_host = re.search(host_re, host) if match_host: host_spec = host_spec._replace(hostname=match_host.group(1)) name_match = re.search(name_re, host) if name_match: host_spec = host_spec._replace(name=name_match.group(1)) ip_match = re.search(ip_re, host) if ip_match: host_spec = host_spec._replace(network=ip_match.group(1)) if not require_network: return host_spec from ipaddress import ip_network, ip_address networks = list() # type: List[str] network = host_spec.network # in case we have [v2:1.2.3.4:3000,v1:1.2.3.4:6478] if ',' in network: networks = [x for x in network.split(',')] else: if network != '': networks.append(network) for network in networks: # only if we have versioned network configs if network.startswith('v') or network.startswith('[v'): # if this is ipv6 we can't just simply split on ':' so do # a split once and rsplit once to leave us with just ipv6 addr network = network.split(':', 1)[1] network = network.rsplit(':', 1)[0] try: # if subnets are defined, also verify the validity if '/' in network: ip_network(six.text_type(network)) else: ip_address(unwrap_ipv6(network)) except ValueError as e: # logging? raise e host_spec.validate() return host_spec def validate(self): assert_valid_host(self.hostname) class PlacementSpec(object): """ For APIs that need to specify a host subset """ def __init__(self, label=None, # type: Optional[str] hosts=None, # type: Union[List[str],List[HostPlacementSpec]] count=None, # type: Optional[int] host_pattern=None # type: Optional[str] ): # type: (...) -> None self.label = label self.hosts = [] # type: List[HostPlacementSpec] if hosts: self.set_hosts(hosts) self.count = count # type: Optional[int] #: fnmatch patterns to select hosts. Can also be a single host. self.host_pattern = host_pattern # type: Optional[str] self.validate() def is_empty(self): return self.label is None and \ not self.hosts and \ not self.host_pattern and \ self.count is None def __eq__(self, other): if isinstance(other, PlacementSpec): return self.label == other.label \ and self.hosts == other.hosts \ and self.count == other.count \ and self.host_pattern == other.host_pattern return NotImplemented def set_hosts(self, hosts): # To backpopulate the .hosts attribute when using labels or count # in the orchestrator backend. if all([isinstance(host, HostPlacementSpec) for host in hosts]): self.hosts = hosts # type: ignore else: self.hosts = [HostPlacementSpec.parse(x, require_network=False) # type: ignore for x in hosts if x] # deprecated def filter_matching_hosts(self, _get_hosts_func: Callable) -> List[str]: return self.filter_matching_hostspecs(_get_hosts_func(as_hostspec=True)) def filter_matching_hostspecs(self, hostspecs: Iterable[HostSpec]) -> List[str]: if self.hosts: all_hosts = [hs.hostname for hs in hostspecs] return [h.hostname for h in self.hosts if h.hostname in all_hosts] elif self.label: return [hs.hostname for hs in hostspecs if self.label in hs.labels] elif self.host_pattern: all_hosts = [hs.hostname for hs in hostspecs] return fnmatch.filter(all_hosts, self.host_pattern) else: # This should be caught by the validation but needs to be here for # get_host_selection_size return [] def get_host_selection_size(self, hostspecs: Iterable[HostSpec]): if self.count: return self.count return len(self.filter_matching_hostspecs(hostspecs)) def pretty_str(self): """ >>> #doctest: +SKIP ... ps = PlacementSpec(...) # For all placement specs: ... PlacementSpec.from_string(ps.pretty_str()) == ps """ kv = [] if self.hosts: kv.append(';'.join([str(h) for h in self.hosts])) if self.count: kv.append('count:%d' % self.count) if self.label: kv.append('label:%s' % self.label) if self.host_pattern: kv.append(self.host_pattern) return ';'.join(kv) def __repr__(self): kv = [] if self.count: kv.append('count=%d' % self.count) if self.label: kv.append('label=%s' % repr(self.label)) if self.hosts: kv.append('hosts={!r}'.format(self.hosts)) if self.host_pattern: kv.append('host_pattern={!r}'.format(self.host_pattern)) return "PlacementSpec(%s)" % ', '.join(kv) @classmethod @handle_type_error def from_json(cls, data): c = data.copy() hosts = c.get('hosts', []) if hosts: c['hosts'] = [] for host in hosts: c['hosts'].append(HostPlacementSpec.from_json(host)) _cls = cls(**c) _cls.validate() return _cls def to_json(self): r = {} if self.label: r['label'] = self.label if self.hosts: r['hosts'] = [host.to_json() for host in self.hosts] if self.count: r['count'] = self.count if self.host_pattern: r['host_pattern'] = self.host_pattern return r def validate(self): if self.hosts and self.label: # TODO: a less generic Exception raise ServiceSpecValidationError('Host and label are mutually exclusive') if self.count is not None and self.count <= 0: raise ServiceSpecValidationError("num/count must be > 1") if self.host_pattern and self.hosts: raise ServiceSpecValidationError('cannot combine host patterns and hosts') for h in self.hosts: h.validate() @classmethod def from_string(cls, arg): # type: (Optional[str]) -> PlacementSpec """ A single integer is parsed as a count: >>> PlacementSpec.from_string('3') PlacementSpec(count=3) A list of names is parsed as host specifications: >>> PlacementSpec.from_string('host1 host2') PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacemen\ tSpec(hostname='host2', network='', name='')]) You can also prefix the hosts with a count as follows: >>> PlacementSpec.from_string('2 host1 host2') PlacementSpec(count=2, hosts=[HostPlacementSpec(hostname='host1', network='', name=''), Hos\ tPlacementSpec(hostname='host2', network='', name='')]) You can specify labels using `label: