# Copyright (C) 2012 Canonical Ltd.
# Copyright (C) 2012 Yahoo! Inc.
#
# Author: Scott Moser <scott.moser@canonical.com>
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
#
# This file is part of cloud-init. See LICENSE file for license information.
import abc
import base64
import copy
import functools
import os
from cloudinit import ec2_utils
from cloudinit import log as logging
from cloudinit import net
from cloudinit import sources
from cloudinit import subp
from cloudinit import url_helper
from cloudinit import util
from cloudinit.sources import BrokenMetadata
# See https://docs.openstack.org/user-guide/cli-config-drive.html
LOG = logging.getLogger(__name__)
FILES_V1 = {
# Path <-> (metadata key name, translator function, default value)
'etc/network/interfaces': ('network_config', lambda x: x, ''),
'meta.js': ('meta_js', util.load_json, {}),
"root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''),
}
KEY_COPIES = (
# Cloud-init metadata names <-> (metadata key, is required)
('local-hostname', 'hostname', False),
('instance-id', 'uuid', True),
)
# Versions and names taken from nova source nova/api/metadata/base.py
OS_LATEST = 'latest'
OS_FOLSOM = '2012-08-10'
OS_GRIZZLY = '2013-04-04'
OS_HAVANA = '2013-10-17'
OS_LIBERTY = '2015-10-15'
# NEWTON_ONE adds 'devices' to md (sriov-pf-passthrough-neutron-port-vlan)
OS_NEWTON_ONE = '2016-06-30'
# NEWTON_TWO adds vendor_data2.json (vendordata-reboot)
OS_NEWTON_TWO = '2016-10-06'
# OS_OCATA adds 'vif' field to devices (sriov-pf-passthrough-neutron-port-vlan)
OS_OCATA = '2017-02-22'
# OS_ROCKY adds a vf_trusted field to devices (sriov-trusted-vfs)
OS_ROCKY = '2018-08-27'
# keep this in chronological order. new supported versions go at the end.
OS_VERSIONS = (
OS_FOLSOM,
OS_GRIZZLY,
OS_HAVANA,
OS_LIBERTY,
OS_NEWTON_ONE,
OS_NEWTON_TWO,
OS_OCATA,
OS_ROCKY,
)
KNOWN_PHYSICAL_TYPES = (
None,
'bgpovs', # not present in OpenStack upstream but used on OVH cloud.
'bridge',
'cascading', # not present in OpenStack upstream, used on OpenTelekomCloud
'dvs',
'ethernet',
'hw_veb',
'hyperv',
'ovs',
'phy',
'tap',
'vhostuser',
'vif',
)
class NonReadable(IOError):
pass
class SourceMixin(object):
def _ec2_name_to_device(self, name):
if not self.ec2_metadata:
return None
bdm = self.ec2_metadata.get('block-device-mapping', {})
for (ent_name, device) in bdm.items():
if name == ent_name:
return device
return None
def get_public_ssh_keys(self):
name = "public_keys"
if self.version == 1:
name = "public-keys"
return sources.normalize_pubkey_data(self.metadata.get(name))
def _os_name_to_device(self, name):
device = None
try:
criteria = 'LABEL=%s' % (name)
if name == 'swap':
criteria = 'TYPE=%s' % (name)
dev_entries = util.find_devs_with(criteria)
if dev_entries:
device = dev_entries[0]
except subp.ProcessExecutionError:
pass
return device
def _validate_device_name(self, device):
if not device:
return None
if not device.startswith("/"):
device = "/dev/%s" % device
if os.path.exists(device):
return device
# Durn, try adjusting the mapping
remapped = self._remap_device(os.path.basename(device))
if remapped:
LOG.debug("Remapped device name %s => %s", device, remapped)
return remapped
return None
def device_name_to_device(self, name):
# Translate a 'name' to a 'physical' device
if not name:
return None
# Try the ec2 mapping first
names = [name]
if name == 'root':
names.insert(0, 'ami')
if name == 'ami':
names.append('root')
device = None
LOG.debug("Using ec2 style lookup to find device %s", names)
for n in names:
device = self._ec2_name_to_device(n)
device = self._validate_device_name(device)
if device:
break
# Try the openstack way second
if not device:
LOG.debug("Using openstack style lookup to find device %s", names)
for n in names:
device = self._os_name_to_device(n)
device = self._validate_device_name(device)
if device:
break
# Ok give up...
if not device:
return None
else:
LOG.debug("Mapped %s to device %s", name, device)
return device
class BaseReader(metaclass=abc.ABCMeta):
def __init__(self, base_path):
self.base_path = base_path
@abc.abstractmethod
def _path_join(self, base, *add_ons):
pass
@abc.abstractmethod
def _path_read(self, path, decode=False):
pass
@abc.abstractmethod
def _fetch_available_versions(self):
pass
@abc.abstractmethod
def _read_ec2_metadata(self):
pass
def _find_working_version(self):
try:
versions_available = self._fetch_available_versions()
except Exception as e:
LOG.debug("Unable to read openstack versions from %s due to: %s",
self.base_path, e)
versions_available = []
# openstack.OS_VERSIONS is stored in chronological order, so
# reverse it to check newest first.
supported = [v for v in reversed(list(OS_VERSIONS))]
selected_version = OS_LATEST
for potential_version in supported:
if potential_version not in versions_available:
continue
selected_version = potential_version
break
LOG.debug("Selected version '%s' from %s", selected_version,
versions_available)
return selected_version
def _read_content_path(self, item, decode=False):
path = item.get('content_path', '').lstrip("/")
path_pieces = path.split("/")
valid_pieces = [p for p in path_pieces if len(p)]
if not valid_pieces:
raise BrokenMetadata("Item %s has no valid content path" % (item))
path = self._path_join(self.base_path, "openstack", *path_pieces)
return self._path_read(path, decode=decode)
def read_v2(self):
"""Reads a version 2 formatted location.
Return a dict with metadata, userdata, ec2-metadata, dsmode,
network_config, files and version (2).
If not a valid location, raise a NonReadable exception.
"""
load_json_anytype = functools.partial(
util.load_json, root_types=(dict, list, str))
def datafiles(version):
files = {}
files['metadata'] = (
# File path to read
self._path_join("openstack", version, 'meta_data.json'),
# Is it required?
True,
# Translator function (applied after loading)
util.load_json,
)
files['userdata'] = (
self._path_join("openstack", version, 'user_data'),
False,
lambda x: x,
)
files['vendordata'] = (
self._path_join("openstack", version, 'vendor_data.json'),
False,
load_json_anytype,
)
files['networkdata'] = (
self._path_join("openstack", version, 'network_data.json'),
False,
load_json_anytype,
)
return files
results = {
'userdata': '',
'version': 2,
}
data = datafiles(self._find_working_version())
for (name, (path, required, translator)) in data.items():
path = self._path_join(self.base_path, path)
data = None
found = False
try:
data = self._path_read(path)
except IOError as e:
if not required:
LOG.debug("Failed reading optional path %s due"
" to: %s", path, e)
else:
LOG.debug("Failed reading mandatory path %s due"
" to: %s", path, e)
else:
found = True
if required and not found:
raise NonReadable("Missing mandatory path: %s" % path)
if found and translator:
try:
data = translator(data)
except Exception as e:
raise BrokenMetadata(
"Failed to process path %s: %s" % (path, e)
) from e
if found:
results[name] = data
metadata = results['metadata']
if 'random_seed' in metadata:
random_seed = metadata['random_seed']
try:
metadata['random_seed'] = base64.b64decode(random_seed)
except (ValueError, TypeError) as e:
raise BrokenMetadata(
"Badly formatted metadata random_seed entry: %s" % e
) from e
# load any files that were provided
files = {}
metadata_files = metadata.get('files', [])
for item in metadata_files:
if 'path' not in item:
continue
path = item['path']
try:
files[path] = self._read_content_path(item)
except Exception as e:
raise BrokenMetadata(
"Failed to read provided file %s: %s" % (path, e)
) from e
results['files'] = files
# The 'network_config' item in metadata is a content pointer
# to the network config that should be applied. It is just a
# ubuntu/debian '/etc/network/interfaces' file.
net_item = metadata.get("network_config", None)
if net_item:
try:
content = self._read_content_path(net_item, decode=True)
results['network_config'] = content
except IOError as e:
raise BrokenMetadata(
"Failed to read network configuration: %s" % (e)
) from e
# To openstack, user can specify meta ('nova boot --meta=key=value')
# and those will appear under metadata['meta'].
# if they specify 'dsmode' they're indicating the mode that they intend
# for this datasource to operate in.
try:
results['dsmode'] = metadata['meta']['dsmode']
except KeyError:
pass
# Read any ec2-metadata (if applicable)
results['ec2-metadata'] = self._read_ec2_metadata()
# Perform some misc. metadata key renames...
for (target_key, source_key, is_required) in KEY_COPIES:
if is_required and source_key not in metadata:
raise BrokenMetadata("No '%s' entry in metadata" % source_key)
if source_key in metadata:
metadata[target_key] = metadata.get(source_key)
return results
class ConfigDriveReader(BaseReader):
def __init__(self, base_path):
super(ConfigDriveReader, self).__init__(base_path)
self._versions = None
def _path_join(self, base, *add_ons):
components = [base] + list(add_ons)
return os.path.join(*components)
def _path_read(self, path, decode=False):
return util.load_file(path, decode=decode)
def _fetch_available_versions(self):
if self._versions is None:
path = self._path_join(self.base_path, 'openstack')
found = [d for d in os.listdir(path)
if os.path.isdir(os.path.join(path))]
self._versions = sorted(found)
return self._versions
def _read_ec2_metadata(self):
path = self._path_join(self.base_path,
'ec2', 'latest', 'meta-data.json')
if not os.path.exists(path):
return {}
else:
try:
return util.load_json(self._path_read(path))
except Exception as e:
raise BrokenMetadata(
"Failed to process path %s: %s" % (path, e)
) from e
def read_v1(self):
"""Reads a version 1 formatted location.
Return a dict with metadata, userdata, dsmode, files and version (1).
If not a valid path, raise a NonReadable exception.
"""
found = {}
for name in FILES_V1.keys():
path = self._path_join(self.base_path, name)
if os.path.exists(path):
found[name] = path
if len(found) == 0:
raise NonReadable("%s: no files found" % (self.base_path))
md = {}
for (name, (key, translator, default)) in FILES_V1.items():
if name in found:
path = found[name]
try:
contents = self._path_read(path)
except IOError as e:
raise BrokenMetadata("Failed to read: %s" % path) from e
try:
# Disable not-callable pylint check; pylint isn't able to
# determine that every member of FILES_V1 has a callable in
# the appropriate position
md[key] = translator(contents) # pylint: disable=E1102
except Exception as e:
raise BrokenMetadata(
"Failed to process path %s: %s" % (path, e)
) from e
else:
md[key] = copy.deepcopy(default)
keydata = md['authorized_keys']
meta_js = md['meta_js']
# keydata in meta_js is preferred over "injected"
keydata = meta_js.get('public-keys', keydata)
if keydata:
lines = keydata.splitlines()
md['public-keys'] = [
line
for line in lines
if len(line) and not line.startswith("#")
]
# config-drive-v1 has no way for openstack to provide the instance-id
# so we copy that into metadata from the user input
if 'instance-id' in meta_js:
md['instance-id'] = meta_js['instance-id']
results = {
'version': 1,
'metadata': md,
}
# allow the user to specify 'dsmode' in a meta tag
if 'dsmode' in meta_js:
results['dsmode'] = meta_js['dsmode']
# config-drive-v1 has no way of specifying user-data, so the user has
# to cheat and stuff it in a meta tag also.
results['userdata'] = meta_js.get('user-data', '')
# this implementation does not support files other than
# network/interfaces and authorized_keys...
results['files'] = {}
return results
class MetadataReader(BaseReader):
def __init__(self, base_url, ssl_details=None, timeout=5, retries=5):
super(MetadataReader, self).__init__(base_url)
self.ssl_details = ssl_details
self.timeout = float(timeout)
self.retries = int(retries)
self._versions = None
def _fetch_available_versions(self):
# <baseurl>/openstack/ returns a newline separated list of versions
if self._versions is not None:
return self._versions
found = []
version_path = self._path_join(self.base_path, "openstack")
content = self._path_read(version_path, decode=True)
for line in content.splitlines():
line = line.strip()
if not line:
continue
found.append(line)
self._versions = found
return self._versions
def _path_read(self, path, decode=False):
def should_retry_cb(_request_args, cause):
try:
code = int(cause.code)
if code >= 400:
return False
except (TypeError, ValueError):
# Older versions of requests didn't have a code.
pass
return True
response = url_helper.readurl(path,
retries=self.retries,
ssl_details=self.ssl_details,
timeout=self.timeout,
exception_cb=should_retry_cb)
if decode:
return response.contents.decode()
else:
return response.contents
def _path_join(self, base, *add_ons):
return url_helper.combine_url(base, *add_ons)
def _read_ec2_metadata(self):
return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details,
timeout=self.timeout,
retries=self.retries)
# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
def convert_net_json(network_json=None, known_macs=None):
"""Return a dictionary of network_config by parsing provided
OpenStack ConfigDrive NetworkData json format
OpenStack network_data.json provides a 3 element dictionary
- "links" (links are network devices, physical or virtual)
- "networks" (networks are ip network configurations for one or more
links)
- services (non-ip services, like dns)
networks and links are combined via network items referencing specific
links via a 'link_id' which maps to a links 'id' field.
To convert this format to network_config yaml, we first iterate over the
links and then walk the network list to determine if any of the networks
utilize the current link; if so we generate a subnet entry for the device
We also need to map network_data.json fields to network_config fields. For
example, the network_data links 'id' field is equivalent to network_config
'name' field for devices. We apply more of this mapping to the various
link types that we encounter.
There are additional fields that are populated in the network_data.json
from OpenStack that are not relevant to network_config yaml, so we
enumerate a dictionary of valid keys for network_yaml and apply filtering
to drop these superflous keys from the network_config yaml.
"""
if network_json is None:
return None
# dict of network_config key for filtering network_json
valid_keys = {
'physical': [
'name',
'type',
'mac_address',
'subnets',
'params',
'mtu',
],
'subnet': [
'type',
'address',
'netmask',
'broadcast',
'metric',
'gateway',
'pointopoint',
'scope',
'dns_nameservers',
'dns_search',
'routes',
],
}
links = network_json.get('links', [])
networks = network_json.get('networks', [])
services = network_json.get('services', [])
link_updates = []
link_id_info = {}
bond_name_fmt = "bond%d"
bond_number = 0
config = []
for link in links:
subnets = []
cfg = dict((k, v) for k, v in link.items()
if k in valid_keys['physical'])
# 'name' is not in openstack spec yet, but we will support it if it is
# present. The 'id' in the spec is currently implemented as the host
# nic's name, meaning something like 'tap-adfasdffd'. We do not want
# to name guest devices with such ugly names.
if 'name' in link:
cfg['name'] = link['name']
link_mac_addr = None
if link.get('ethernet_mac_address'):
link_mac_addr = link.get('ethernet_mac_address').lower()
link_id_info[link['id']] = link_mac_addr
curinfo = {'name': cfg.get('name'), 'mac': link_mac_addr,
'id': link['id'], 'type': link['type']}
for network in [n for n in networks
if n['link'] == link['id']]:
subnet = dict((k, v) for k, v in network.items()
if k in valid_keys['subnet'])
if network['type'] == 'ipv4_dhcp':
subnet.update({'type': 'dhcp4'})
elif network['type'] == 'ipv6_dhcp':
subnet.update({'type': 'dhcp6'})
elif network['type'] in ['ipv6_slaac', 'ipv6_dhcpv6-stateless',
'ipv6_dhcpv6-stateful']:
subnet.update({'type': network['type']})
elif network['type'] in ['ipv4', 'static']:
subnet.update({
'type': 'static',
'address': network.get('ip_address'),
})
elif network['type'] in ['ipv6', 'static6']:
cfg.update({'accept-ra': False})
subnet.update({
'type': 'static6',
'address': network.get('ip_address'),
})
# Enable accept_ra for stateful and legacy ipv6_dhcp types
if network['type'] in ['ipv6_dhcpv6-stateful', 'ipv6_dhcp']:
cfg.update({'accept-ra': True})
if network['type'] == 'ipv4':
subnet['ipv4'] = True
if network['type'] == 'ipv6':
subnet['ipv6'] = True
subnets.append(subnet)
cfg.update({'subnets': subnets})
if link['type'] in ['bond']:
params = {}
if link_mac_addr:
params['mac_address'] = link_mac_addr
for k, v in link.items():
if k == 'bond_links':
continue
elif k.startswith('bond'):
params.update({k: v})
# openstack does not provide a name for the bond.
# they do provide an 'id', but that is possibly non-sensical.
# so we just create our own name.
link_name = bond_name_fmt % bond_number
bond_number += 1
# bond_links reference links by their id, but we need to add
# to the network config by their nic name.
# store that in bond_links_needed, and update these later.
link_updates.append(
(cfg, 'bond_interfaces', '%s',
copy.deepcopy(link['bond_links']))
)
cfg.update({'params': params, 'name': link_name})
curinfo['name'] = link_name
elif link['type'] in ['vlan']:
name = "%s.%s" % (link['vlan_link'], link['vlan_id'])
cfg.update({
'name': name,
'vlan_id': link['vlan_id'],
'mac_address': link['vlan_mac_address'],
})
link_updates.append((cfg, 'vlan_link', '%s', link['vlan_link']))
link_updates.append((cfg, 'name', "%%s.%s" % link['vlan_id'],
link['vlan_link']))
curinfo.update({'mac': link['vlan_mac_address'],
'name': name})
else:
if link['type'] not in KNOWN_PHYSICAL_TYPES:
LOG.warning('Unknown network_data link type (%s); treating as'
' physical', link['type'])
cfg.update({'type': 'physical', 'mac_address': link_mac_addr})
config.append(cfg)
link_id_info[curinfo['id']] = curinfo
need_names = [d for d in config
if d.get('type') == 'physical' and 'name' not in d]
if need_names or link_updates:
if known_macs is None:
known_macs = net.get_interfaces_by_mac()
# go through and fill out the link_id_info with names
for _link_id, info in link_id_info.items():
if info.get('name'):
continue
if info.get('mac') in known_macs:
info['name'] = known_macs[info['mac']]
for d in need_names:
mac = d.get('mac_address')
if not mac:
raise ValueError("No mac_address or name entry for %s" % d)
if mac not in known_macs:
raise ValueError("Unable to find a system nic for %s" % d)
d['name'] = known_macs[mac]
for cfg, key, fmt, targets in link_updates:
if isinstance(targets, (list, tuple)):
cfg[key] = [
fmt % link_id_info[target]['name'] for target in targets
]
else:
cfg[key] = fmt % link_id_info[targets]['name']
# Infiniband interfaces may be referenced in network_data.json by a 6 byte
# Ethernet MAC-style address, and we use that address to look up the
# interface name above. Now ensure that the hardware address is set to the
# full 20 byte address.
ib_known_hwaddrs = net.get_ib_hwaddrs_by_interface()
if ib_known_hwaddrs:
for cfg in config:
if cfg['name'] in ib_known_hwaddrs:
cfg['mac_address'] = ib_known_hwaddrs[cfg['name']]
cfg['type'] = 'infiniband'
for service in services:
cfg = service
cfg.update({'type': 'nameserver'})
config.append(cfg)
return {'version': 1, 'config': config}
# vi: ts=4 expandtab