Print this page
Handle routes_data better
Handle 'routes' being empty (missing bits)
Handle 'routes' being empty
IMAGE-1097 cloud-init doesn't use sdc:routes metadata
| Split |
Close |
| Expand all |
| Collapse all |
--- old/cloudinit/sources/DataSourceSmartOS.py
+++ new/cloudinit/sources/DataSourceSmartOS.py
1 1 # Copyright (C) 2013 Canonical Ltd.
2 2 # Copyright (c) 2018, Joyent, Inc.
3 3 #
4 4 # Author: Ben Howard <ben.howard@canonical.com>
5 5 #
6 6 # This file is part of cloud-init. See LICENSE file for license information.
7 7
8 8 # Datasource for provisioning on SmartOS. This works on Joyent
9 9 # and public/private Clouds using SmartOS.
10 10 #
11 11 # SmartOS hosts use a serial console (/dev/ttyS1) on KVM Linux Guests
12 12 # The meta-data is transmitted via key/value pairs made by
13 13 # requests on the console. For example, to get the hostname, you
14 14 # would send "GET hostname" on /dev/ttyS1.
15 15 # For Linux Guests running in LX-Brand Zones on SmartOS hosts
16 16 # a socket (/native/.zonecontrol/metadata.sock) is used instead
17 17 # of a serial console.
18 18 #
19 19 # Certain behavior is defined by the DataDictionary
20 20 # http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html
21 21 # Comments with "@datadictionary" are snippets of the definition
22 22
23 23 import base64
24 24 import binascii
25 25 import errno
26 26 import fcntl
27 27 import json
28 28 import os
29 29 import random
30 30 import re
31 31 import socket
32 32 import time
33 33
34 34 from cloudinit import log as logging
35 35 from cloudinit import serial
36 36 from cloudinit import sources
37 37 from cloudinit import util
38 38
39 39 LOG = logging.getLogger(__name__)
40 40
41 41 SMARTOS_ATTRIB_MAP = {
42 42 # Cloud-init Key : (SmartOS Key, Strip line endings)
43 43 'instance-id': ('sdc:uuid', True),
44 44 'local-hostname': ('hostname', True),
45 45 'public-keys': ('root_authorized_keys', True),
46 46 'user-script': ('user-script', False),
47 47 'legacy-user-data': ('user-data', False),
48 48 'user-data': ('cloud-init:user-data', False),
49 49 'iptables_disable': ('iptables_disable', True),
50 50 'motd_sys_info': ('motd_sys_info', True),
51 51 'availability_zone': ('sdc:datacenter_name', True),
52 52 'vendor-data': ('sdc:vendor-data', False),
53 53 'operator-script': ('sdc:operator-script', False),
54 54 'hostname': ('sdc:hostname', True),
55 55 'dns_domain': ('sdc:dns_domain', True),
56 56 }
57 57
58 58 SMARTOS_ATTRIB_JSON = {
59 59 # Cloud-init Key : (SmartOS Key known JSON)
60 60 'network-data': 'sdc:nics',
61 61 'dns_servers': 'sdc:resolvers',
62 62 'routes': 'sdc:routes',
63 63 }
64 64
65 65 SMARTOS_ENV_LX_BRAND = "lx-brand"
66 66 SMARTOS_ENV_KVM = "kvm"
67 67
68 68 DS_NAME = 'SmartOS'
69 69 DS_CFG_PATH = ['datasource', DS_NAME]
70 70 NO_BASE64_DECODE = [
71 71 'iptables_disable',
72 72 'motd_sys_info',
73 73 'root_authorized_keys',
74 74 'sdc:datacenter_name',
75 75 'sdc:uuid'
76 76 'user-data',
77 77 'user-script',
78 78 ]
79 79
80 80 METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock'
81 81 SERIAL_DEVICE = '/dev/ttyS1'
82 82 SERIAL_TIMEOUT = 60
83 83
84 84 # BUILT-IN DATASOURCE CONFIGURATION
85 85 # The following is the built-in configuration. If the values
86 86 # are not set via the system configuration, then these default
87 87 # will be used:
88 88 # serial_device: which serial device to use for the meta-data
89 89 # serial_timeout: how long to wait on the device
90 90 # no_base64_decode: values which are not base64 encoded and
91 91 # are fetched directly from SmartOS, not meta-data values
92 92 # base64_keys: meta-data keys that are delivered in base64
93 93 # base64_all: with the exclusion of no_base64_decode values,
94 94 # treat all meta-data as base64 encoded
95 95 # disk_setup: describes how to partition the ephemeral drive
96 96 # fs_setup: describes how to format the ephemeral drive
97 97 #
98 98 BUILTIN_DS_CONFIG = {
99 99 'serial_device': SERIAL_DEVICE,
100 100 'serial_timeout': SERIAL_TIMEOUT,
101 101 'metadata_sockfile': METADATA_SOCKFILE,
102 102 'no_base64_decode': NO_BASE64_DECODE,
103 103 'base64_keys': [],
104 104 'base64_all': False,
105 105 'disk_aliases': {'ephemeral0': '/dev/vdb'},
106 106 }
107 107
108 108 BUILTIN_CLOUD_CONFIG = {
109 109 'disk_setup': {
110 110 'ephemeral0': {'table_type': 'mbr',
111 111 'layout': False,
112 112 'overwrite': False}
113 113 },
114 114 'fs_setup': [{'label': 'ephemeral0',
115 115 'filesystem': 'ext3',
116 116 'device': 'ephemeral0'}],
117 117 }
118 118
119 119 # builtin vendor-data is a boothook that writes a script into
120 120 # /var/lib/cloud/scripts/per-boot. *That* script then handles
121 121 # executing the 'operator-script' and 'user-script' files
122 122 # that cloud-init writes into /var/lib/cloud/instance/data/
123 123 # if they exist.
124 124 #
125 125 # This is all very indirect, but its done like this so that at
126 126 # some point in the future, perhaps cloud-init wouldn't do it at
127 127 # all, but rather the vendor actually provide vendor-data that accomplished
128 128 # their desires. (That is the point of vendor-data).
129 129 #
130 130 # cloud-init does cheat a bit, and write the operator-script and user-script
131 131 # itself. It could have the vendor-script do that, but it seems better
132 132 # to not require the image to contain a tool (mdata-get) to read those
133 133 # keys when we have a perfectly good one inside cloud-init.
134 134 BUILTIN_VENDOR_DATA = """\
135 135 #cloud-boothook
136 136 #!/bin/sh
137 137 fname="%(per_boot_d)s/01_smartos_vendor_data.sh"
138 138 mkdir -p "${fname%%/*}"
139 139 cat > "$fname" <<"END_SCRIPT"
140 140 #!/bin/sh
141 141 ##
142 142 # This file is written as part of the default vendor data for SmartOS.
143 143 # The SmartOS datasource writes the listed file from the listed metadata key
144 144 # sdc:operator-script -> %(operator_script)s
145 145 # user-script -> %(user_script)s
146 146 #
147 147 # You can view content with 'mdata-get <key>'
148 148 #
149 149 for script in "%(operator_script)s" "%(user_script)s"; do
150 150 [ -x "$script" ] || continue
151 151 echo "executing '$script'" 1>&2
152 152 "$script"
153 153 done
154 154 END_SCRIPT
155 155 chmod +x "$fname"
156 156 """
157 157
158 158
159 159 # @datadictionary: this is legacy path for placing files from metadata
160 160 # per the SmartOS location. It is not preferable, but is done for
161 161 # legacy reasons
162 162 LEGACY_USER_D = "/var/db"
163 163
164 164
165 165 class DataSourceSmartOS(sources.DataSource):
166 166
167 167 dsname = "Joyent"
168 168
169 169 _unset = "_unset"
170 170 smartos_type = _unset
171 171 md_client = _unset
172 172
173 173 def __init__(self, sys_cfg, distro, paths):
174 174 sources.DataSource.__init__(self, sys_cfg, distro, paths)
175 175 self.ds_cfg = util.mergemanydict([
176 176 self.ds_cfg,
177 177 util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}),
178 178 BUILTIN_DS_CONFIG])
179 179
180 180 self.metadata = {}
181 181 self.network_data = None
182 182 self._network_config = None
183 183
184 184 self.script_base_d = os.path.join(self.paths.get_cpath("scripts"))
185 185
186 186 self._init()
187 187
188 188 def __str__(self):
189 189 root = sources.DataSource.__str__(self)
190 190 return "%s [client=%s]" % (root, self.md_client)
191 191
192 192 def _init(self):
193 193 if self.smartos_type == self._unset:
194 194 self.smartos_type = get_smartos_environ()
195 195 if self.smartos_type is None:
196 196 self.md_client = None
197 197
198 198 if self.md_client == self._unset:
199 199 self.md_client = jmc_client_factory(
200 200 smartos_type=self.smartos_type,
201 201 metadata_sockfile=self.ds_cfg['metadata_sockfile'],
202 202 serial_device=self.ds_cfg['serial_device'],
203 203 serial_timeout=self.ds_cfg['serial_timeout'])
204 204
205 205 def _set_provisioned(self):
206 206 '''Mark the instance provisioning state as successful.
207 207
208 208 When run in a zone, the host OS will look for /var/svc/provisioning
209 209 to be renamed as /var/svc/provision_success. This should be done
210 210 after meta-data is successfully retrieved and from this point
211 211 the host considers the provision of the zone to be a success and
212 212 keeps the zone running.
213 213 '''
214 214
215 215 LOG.debug('Instance provisioning state set as successful')
216 216 svc_path = '/var/svc'
217 217 if os.path.exists('/'.join([svc_path, 'provisioning'])):
218 218 os.rename('/'.join([svc_path, 'provisioning']),
219 219 '/'.join([svc_path, 'provision_success']))
220 220
221 221 def _get_data(self):
222 222 self._init()
223 223
224 224 md = {}
225 225 ud = ""
226 226
227 227 if not self.smartos_type:
228 228 LOG.debug("Not running on smartos")
229 229 return False
230 230
231 231 if not self.md_client.exists():
232 232 LOG.debug("No metadata device '%r' found for SmartOS datasource",
233 233 self.md_client)
234 234 return False
235 235
236 236 # Open once for many requests, rather than once for each request
237 237 self.md_client.open_transport()
238 238
239 239 for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items():
240 240 smartos_noun, strip = attribute
241 241 md[ci_noun] = self.md_client.get(smartos_noun, strip=strip)
242 242
243 243 for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items():
244 244 md[ci_noun] = self.md_client.get_json(smartos_noun)
245 245
246 246 self.md_client.close_transport()
247 247
248 248 # @datadictionary: This key may contain a program that is written
249 249 # to a file in the filesystem of the guest on each boot and then
250 250 # executed. It may be of any format that would be considered
251 251 # executable in the guest instance.
252 252 #
253 253 # We write 'user-script' and 'operator-script' into the
254 254 # instance/data directory. The default vendor-data then handles
255 255 # executing them later.
256 256 data_d = os.path.join(self.paths.get_cpath(), 'instances',
257 257 md['instance-id'], 'data')
258 258 user_script = os.path.join(data_d, 'user-script')
259 259 u_script_l = "%s/user-script" % LEGACY_USER_D
260 260 write_boot_content(md.get('user-script'), content_f=user_script,
261 261 link=u_script_l, shebang=True, mode=0o700)
262 262
263 263 operator_script = os.path.join(data_d, 'operator-script')
264 264 write_boot_content(md.get('operator-script'),
265 265 content_f=operator_script, shebang=False,
266 266 mode=0o700)
267 267
268 268 # @datadictionary: This key has no defined format, but its value
269 269 # is written to the file /var/db/mdata-user-data on each boot prior
270 270 # to the phase that runs user-script. This file is not to be executed.
271 271 # This allows a configuration file of some kind to be injected into
272 272 # the machine to be consumed by the user-script when it runs.
273 273 u_data = md.get('legacy-user-data')
274 274 u_data_f = "%s/mdata-user-data" % LEGACY_USER_D
275 275 write_boot_content(u_data, u_data_f)
276 276
277 277 # Handle the cloud-init regular meta
278 278 if not md['local-hostname']:
279 279 md['local-hostname'] = md['instance-id']
280 280
281 281 ud = None
282 282 if md['user-data']:
283 283 ud = md['user-data']
284 284
285 285 if not md['vendor-data']:
286 286 md['vendor-data'] = BUILTIN_VENDOR_DATA % {
|
↓ open down ↓ |
286 lines elided |
↑ open up ↑ |
287 287 'user_script': user_script,
288 288 'operator_script': operator_script,
289 289 'per_boot_d': os.path.join(self.paths.get_cpath("scripts"),
290 290 'per-boot'),
291 291 }
292 292
293 293 self.metadata = util.mergemanydict([md, self.metadata])
294 294 self.userdata_raw = ud
295 295 self.vendordata_raw = md['vendor-data']
296 296 self.network_data = md['network-data']
297 + self.routes_data = md['routes']
297 298
298 299 self._set_provisioned()
299 300 return True
300 301
301 302 def device_name_to_device(self, name):
302 303 return self.ds_cfg['disk_aliases'].get(name)
303 304
304 305 def get_config_obj(self):
305 306 if self.smartos_type == SMARTOS_ENV_KVM:
306 307 return BUILTIN_CLOUD_CONFIG
307 308 return {}
308 309
309 310 def get_instance_id(self):
|
↓ open down ↓ |
3 lines elided |
↑ open up ↑ |
310 311 return self.metadata['instance-id']
311 312
312 313 @property
313 314 def network_config(self):
314 315 if self._network_config is None:
315 316 if self.network_data is not None:
316 317 self._network_config = (
317 318 convert_smartos_network_data(
318 319 network_data=self.network_data,
319 320 dns_servers=self.metadata['dns_servers'],
320 - dns_domain=self.metadata['dns_domain']))
321 + dns_domain=self.metadata['dns_domain'],
322 + routes=self.routes_data))
321 323 return self._network_config
322 324
323 325
324 326 class JoyentMetadataFetchException(Exception):
325 327 pass
326 328
327 329
328 330 class JoyentMetadataTimeoutException(JoyentMetadataFetchException):
329 331 pass
330 332
331 333
332 334 class JoyentMetadataClient(object):
333 335 """
334 336 A client implementing v2 of the Joyent Metadata Protocol Specification.
335 337
336 338 The full specification can be found at
337 339 http://eng.joyent.com/mdata/protocol.html
338 340 """
339 341 line_regex = re.compile(
340 342 r'V2 (?P<length>\d+) (?P<checksum>[0-9a-f]+)'
341 343 r' (?P<body>(?P<request_id>[0-9a-f]+) (?P<status>SUCCESS|NOTFOUND)'
342 344 r'( (?P<payload>.+))?)')
343 345
344 346 def __init__(self, smartos_type=None, fp=None):
345 347 if smartos_type is None:
346 348 smartos_type = get_smartos_environ()
347 349 self.smartos_type = smartos_type
348 350 self.fp = fp
349 351
350 352 def _checksum(self, body):
351 353 return '{0:08x}'.format(
352 354 binascii.crc32(body.encode('utf-8')) & 0xffffffff)
353 355
354 356 def _get_value_from_frame(self, expected_request_id, frame):
355 357 match = self.line_regex.match(frame)
356 358 if match is None:
357 359 raise JoyentMetadataFetchException(
358 360 'No regex match for frame "%s"' % frame)
359 361 frame_data = match.groupdict()
360 362 if int(frame_data['length']) != len(frame_data['body']):
361 363 raise JoyentMetadataFetchException(
362 364 'Incorrect frame length given ({0} != {1}).'.format(
363 365 frame_data['length'], len(frame_data['body'])))
364 366 expected_checksum = self._checksum(frame_data['body'])
365 367 if frame_data['checksum'] != expected_checksum:
366 368 raise JoyentMetadataFetchException(
367 369 'Invalid checksum (expected: {0}; got {1}).'.format(
368 370 expected_checksum, frame_data['checksum']))
369 371 if frame_data['request_id'] != expected_request_id:
370 372 raise JoyentMetadataFetchException(
371 373 'Request ID mismatch (expected: {0}; got {1}).'.format(
372 374 expected_request_id, frame_data['request_id']))
373 375 if not frame_data.get('payload', None):
374 376 LOG.debug('No value found.')
375 377 return None
376 378 value = util.b64d(frame_data['payload'])
377 379 LOG.debug('Value "%s" found.', value)
378 380 return value
379 381
380 382 def _readline(self):
381 383 """
382 384 Reads a line a byte at a time until \n is encountered. Returns an
383 385 ascii string with the trailing newline removed.
384 386
385 387 If a timeout (per-byte) is set and it expires, a
386 388 JoyentMetadataFetchException will be thrown.
387 389 """
388 390 response = bytearray()
389 391 while True:
390 392 try:
391 393 byte = self.fp.read(1)
392 394 if byte == b'\n':
393 395 return response.decode('ascii')
394 396 if byte == b'':
395 397 raise JoyentMetadataTimeoutException(
396 398 "Partial response: '%s'" % response.decode('ascii'))
397 399 response.extend(byte)
398 400 except OSError as exc:
399 401 if exc.errno == errno.EAGAIN:
400 402 raise JoyentMetadataTimeoutException(
401 403 "Partial response: '%s'" % response.decode('ascii'))
402 404 raise
403 405
404 406 def _write(self, msg):
405 407 self.fp.write(msg.encode('ascii'))
406 408 self.fp.flush()
407 409
408 410 def _negotiate(self):
409 411 LOG.debug('Negotiating protocol V2')
410 412 self._write('NEGOTIATE V2\n')
411 413 self.fp.flush()
412 414 response = self._readline()
413 415 if response != 'V2_OK':
414 416 raise JoyentMetadataFetchException(
415 417 'Invalid response "%s" to "NEGOTIATE V2"' % response)
416 418 LOG.debug('Negotiation complete')
417 419
418 420 def request(self, rtype, param=None):
419 421 request_id = '{0:08x}'.format(random.randint(0, 0xffffffff))
420 422 message_body = ' '.join((request_id, rtype,))
421 423 if param:
422 424 message_body += ' ' + base64.b64encode(param.encode()).decode()
423 425 msg = 'V2 {0} {1} {2}\n'.format(
424 426 len(message_body), self._checksum(message_body), message_body)
425 427 LOG.debug('Writing "%s" to metadata transport.', msg)
426 428
427 429 need_close = False
428 430 if not self.fp:
429 431 self.open_transport()
430 432 need_close = True
431 433
432 434 self._write(msg)
433 435
434 436 response = self._readline()
435 437 if need_close:
436 438 self.close_transport()
437 439
438 440 LOG.debug('Read "%s" from metadata transport.', response)
439 441
440 442 if 'SUCCESS' not in response:
441 443 return None
442 444
443 445 value = self._get_value_from_frame(request_id, response)
444 446 return value
445 447
446 448 def get(self, key, default=None, strip=False):
447 449 # Do a couple tries in case someone else has the serial port open
448 450 # before this process opened it. This also helps in the event that
449 451 # the metadata server goes away in middle of a conversation.
450 452 for tries in [1, 2]:
451 453 try:
452 454 result = self.request(rtype='GET', param=key)
453 455 if result is None:
454 456 return default
455 457 if result and strip:
456 458 result = result.strip()
457 459 return result
458 460 except JoyentMetadataFetchException as exc:
459 461 LOG.warning('Try %d: GET "%s" failed: %s', tries, key, exc)
460 462 last_exc = exc
461 463 pass
462 464 raise(last_exc)
463 465
464 466 def get_json(self, key, default=None):
465 467 result = self.get(key, default=default)
466 468 if result is None:
467 469 return default
468 470 return json.loads(result)
469 471
470 472 def list(self):
471 473 result = self.request(rtype='KEYS')
472 474 if not result:
473 475 return []
474 476 return result.split('\n')
475 477
476 478 def put(self, key, val):
477 479 param = b' '.join([base64.b64encode(i.encode())
478 480 for i in (key, val)]).decode()
479 481 return self.request(rtype='PUT', param=param)
480 482
481 483 def delete(self, key):
482 484 return self.request(rtype='DELETE', param=key)
483 485
484 486 def close_transport(self):
485 487 if self.fp:
486 488 self.fp.close()
487 489 self.fp = None
488 490
489 491 def __enter__(self):
490 492 if self.fp:
491 493 return self
492 494 self.open_transport()
493 495 return self
494 496
495 497 def __exit__(self, exc_type, exc_value, traceback):
496 498 self.close_transport()
497 499 return
498 500
499 501 def open_transport(self):
500 502 raise NotImplementedError
501 503
502 504
503 505 class JoyentMetadataSocketClient(JoyentMetadataClient):
504 506 def __init__(self, socketpath, smartos_type=SMARTOS_ENV_LX_BRAND):
505 507 super(JoyentMetadataSocketClient, self).__init__(smartos_type)
506 508 self.socketpath = socketpath
507 509
508 510 def open_transport(self):
509 511 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
510 512 sock.connect(self.socketpath)
511 513 self.fp = sock.makefile('rwb')
512 514 self._negotiate()
513 515
514 516 def exists(self):
515 517 return os.path.exists(self.socketpath)
516 518
517 519 def __repr__(self):
518 520 return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath)
519 521
520 522
521 523 class JoyentMetadataSerialClient(JoyentMetadataClient):
522 524 def __init__(self, device, timeout=10, smartos_type=SMARTOS_ENV_KVM):
523 525 super(JoyentMetadataSerialClient, self).__init__(smartos_type)
524 526 self.device = device
525 527 self.timeout = timeout
526 528
527 529 def exists(self):
528 530 return os.path.exists(self.device)
529 531
530 532 def open_transport(self):
531 533 ser = serial.Serial(self.device, timeout=self.timeout)
532 534 if not ser.isOpen():
533 535 # There is some sort of a race between cloud-init and mdata-get
534 536 # that can cause the serial device to not open on the initial
535 537 # attempt.
536 538 for tries in range(1, 11):
537 539 try:
538 540 ser.open()
539 541 assert(ser.isOpen())
540 542 break
541 543 except OSError as exc:
542 544 # This is probably a SerialException, which is a subclass
543 545 # of OSError. SerialException is not used becasue of
544 546 # existing efforts to make pyserial optional.
545 547 LOG.debug("Failed to open %s on try %d: %s", self.device,
546 548 tries, exc)
547 549 time.sleep(0.1)
548 550 else:
549 551 raise SystemError("Unable to open %s" % self.device)
550 552 fcntl.lockf(ser, fcntl.LOCK_EX)
551 553 self.fp = ser
552 554 self._flush()
553 555 self._negotiate()
554 556
555 557 def bad_readline(self):
556 558 response = self.fp.read_until('\n').decode('ascii')
557 559 if response[-1:] != '\n':
558 560 raise JoyentMetadataTimeoutException(
559 561 "Partial response: b'%s'" % response)
560 562 return(response[0:-1])
561 563
562 564 def _flush(self):
563 565 LOG.debug('Flushing input')
564 566 # Read any pending data
565 567 timeout = self.fp.timeout
566 568 self.fp.timeout = 0.1
567 569 while True:
568 570 try:
569 571 self._readline()
570 572 except JoyentMetadataTimeoutException:
571 573 break
572 574 LOG.debug('Input empty')
573 575
574 576 # Send a newline and expect "invalid command". Keep trying until
575 577 # successful. Retry rather frequently so that the "Is the host
576 578 # metadata service running" appears on the console soon after someone
577 579 # attaches in an effort to debug.
578 580 if timeout > 5:
579 581 self.fp.timeout = 5
580 582 else:
581 583 self.fp.timeout = timeout
582 584 while True:
583 585 LOG.debug('Writing newline, expecting "invalid command"')
584 586 self._write('\n')
585 587 try:
586 588 response = self._readline()
587 589 if response == 'invalid command':
588 590 break
589 591 if response == 'FAILURE':
590 592 LOG.debug('Got "FAILURE". Retrying.');
591 593 continue
592 594 LOG.warning('Unexpected response "%s" during flush', response)
593 595 except JoyentMetadataTimeoutException:
594 596 LOG.warning('Timeout while initializing metadata client. ' +
595 597 'Is the host metadata service running?')
596 598 LOG.debug('Got "invalid command". Flush complete.')
597 599 self.fp.timeout = timeout
598 600
599 601 def __repr__(self):
600 602 return "%s(device=%s, timeout=%s)" % (
601 603 self.__class__.__name__, self.device, self.timeout)
602 604
603 605
604 606 class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient):
605 607 """V1 of the protocol was not safe for all values.
606 608 Thus, we allowed the user to pass values in as base64 encoded.
607 609 Users may still reasonably expect to be able to send base64 data
608 610 and have it transparently decoded. So even though the V2 format is
609 611 now used, and is safe (using base64 itself), we keep legacy support.
610 612
611 613 The way for a user to do this was:
612 614 a.) specify 'base64_keys' key whose value is a comma delimited
613 615 list of keys that were base64 encoded.
614 616 b.) base64_all: string interpreted as a boolean that indicates
615 617 if all keys are base64 encoded.
616 618 c.) set a key named b64-<keyname> with a boolean indicating that
617 619 <keyname> is base64 encoded."""
618 620
619 621 def __init__(self, device, timeout=10, smartos_type=None):
620 622 s = super(JoyentMetadataLegacySerialClient, self)
621 623 s.__init__(device, timeout, smartos_type)
622 624 self.base64_keys = None
623 625 self.base64_all = None
624 626
625 627 def _init_base64_keys(self, reset=False):
626 628 if reset:
627 629 self.base64_keys = None
628 630 self.base64_all = None
629 631
630 632 keys = None
631 633 if self.base64_all is None:
632 634 keys = self.list()
633 635 if 'base64_all' in keys:
634 636 self.base64_all = util.is_true(self._get("base64_all"))
635 637 else:
636 638 self.base64_all = False
637 639
638 640 if self.base64_all:
639 641 # short circuit if base64_all is true
640 642 return
641 643
642 644 if self.base64_keys is None:
643 645 if keys is None:
644 646 keys = self.list()
645 647 b64_keys = set()
646 648 if 'base64_keys' in keys:
647 649 b64_keys = set(self._get("base64_keys").split(","))
648 650
649 651 # now add any b64-<keyname> that has a true value
650 652 for key in [k[3:] for k in keys if k.startswith("b64-")]:
651 653 if util.is_true(self._get(key)):
652 654 b64_keys.add(key)
653 655 else:
654 656 if key in b64_keys:
655 657 b64_keys.remove(key)
656 658
657 659 self.base64_keys = b64_keys
658 660
659 661 def _get(self, key, default=None, strip=False):
660 662 return (super(JoyentMetadataLegacySerialClient, self).
661 663 get(key, default=default, strip=strip))
662 664
663 665 def is_b64_encoded(self, key, reset=False):
664 666 if key in NO_BASE64_DECODE:
665 667 return False
666 668
667 669 self._init_base64_keys(reset=reset)
668 670 if self.base64_all:
669 671 return True
670 672
671 673 return key in self.base64_keys
672 674
673 675 def get(self, key, default=None, strip=False):
674 676 mdefault = object()
675 677 val = self._get(key, strip=False, default=mdefault)
676 678 if val is mdefault:
677 679 return default
678 680
679 681 if self.is_b64_encoded(key):
680 682 try:
681 683 val = base64.b64decode(val.encode()).decode()
682 684 # Bogus input produces different errors in Python 2 and 3
683 685 except (TypeError, binascii.Error):
684 686 LOG.warning("Failed base64 decoding key '%s': %s", key, val)
685 687
686 688 if strip:
687 689 val = val.strip()
688 690
689 691 return val
690 692
691 693
692 694 def jmc_client_factory(
693 695 smartos_type=None, metadata_sockfile=METADATA_SOCKFILE,
694 696 serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT,
695 697 uname_version=None):
696 698
697 699 if smartos_type is None:
698 700 smartos_type = get_smartos_environ(uname_version)
699 701
700 702 if smartos_type is None:
701 703 return None
702 704 elif smartos_type == SMARTOS_ENV_KVM:
703 705 return JoyentMetadataLegacySerialClient(
704 706 device=serial_device, timeout=serial_timeout,
705 707 smartos_type=smartos_type)
706 708 elif smartos_type == SMARTOS_ENV_LX_BRAND:
707 709 return JoyentMetadataSocketClient(socketpath=metadata_sockfile,
708 710 smartos_type=smartos_type)
709 711
710 712 raise ValueError("Unknown value for smartos_type: %s" % smartos_type)
711 713
712 714
713 715 def write_boot_content(content, content_f, link=None, shebang=False,
714 716 mode=0o400):
715 717 """
716 718 Write the content to content_f. Under the following rules:
717 719 1. If no content, remove the file
718 720 2. Write the content
719 721 3. If executable and no file magic, add it
720 722 4. If there is a link, create it
721 723
722 724 @param content: what to write
723 725 @param content_f: the file name
724 726 @param backup_d: the directory to save the backup at
725 727 @param link: if defined, location to create a symlink to
726 728 @param shebang: if no file magic, set shebang
727 729 @param mode: file mode
728 730
729 731 Becuase of the way that Cloud-init executes scripts (no shell),
730 732 a script will fail to execute if does not have a magic bit (shebang) set
731 733 for the file. If shebang=True, then the script will be checked for a magic
732 734 bit and to the SmartOS default of assuming that bash.
733 735 """
734 736
735 737 if not content and os.path.exists(content_f):
736 738 os.unlink(content_f)
737 739 if link and os.path.islink(link):
738 740 os.unlink(link)
739 741 if not content:
740 742 return
741 743
742 744 util.write_file(content_f, content, mode=mode)
743 745
744 746 if shebang and not content.startswith("#!"):
745 747 try:
746 748 cmd = ["file", "--brief", "--mime-type", content_f]
747 749 (f_type, _err) = util.subp(cmd)
748 750 LOG.debug("script %s mime type is %s", content_f, f_type)
749 751 if f_type.strip() == "text/plain":
750 752 new_content = "\n".join(["#!/bin/bash", content])
751 753 util.write_file(content_f, new_content, mode=mode)
752 754 LOG.debug("added shebang to file %s", content_f)
753 755
754 756 except Exception as e:
755 757 util.logexc(LOG, ("Failed to identify script type for %s" %
756 758 content_f, e))
757 759
758 760 if link:
759 761 try:
760 762 if os.path.islink(link):
761 763 os.unlink(link)
762 764 if content and os.path.exists(content_f):
763 765 util.ensure_dir(os.path.dirname(link))
764 766 os.symlink(content_f, link)
765 767 except IOError as e:
766 768 util.logexc(LOG, "failed establishing content link: %s", e)
767 769
768 770
769 771 def get_smartos_environ(uname_version=None, product_name=None):
770 772 uname = os.uname()
771 773
772 774 # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
773 775 # report 'BrandZ virtual linux' as the kernel version
774 776 if uname_version is None:
775 777 uname_version = uname[3]
776 778 if uname_version.lower() == 'brandz virtual linux':
777 779 return SMARTOS_ENV_LX_BRAND
778 780
779 781 if product_name is None:
780 782 system_type = util.read_dmi_data("system-product-name")
781 783 else:
|
↓ open down ↓ |
451 lines elided |
↑ open up ↑ |
782 784 system_type = product_name
783 785
784 786 if system_type and 'smartdc' in system_type.lower():
785 787 return SMARTOS_ENV_KVM
786 788
787 789 return None
788 790
789 791
790 792 # Convert SMARTOS 'sdc:nics' data to network_config yaml
791 793 def convert_smartos_network_data(network_data=None,
792 - dns_servers=None, dns_domain=None):
794 + dns_servers=None, dns_domain=None,
795 + routes=None):
793 796 """Return a dictionary of network_config by parsing provided
794 797 SMARTOS sdc:nics configuration data
795 798
796 799 sdc:nics data is a dictionary of properties of a nic and the ip
797 800 configuration desired. Additional nic dictionaries are appended
798 801 to the list.
799 802
800 803 Converting the format is straightforward though it does include
801 804 duplicate information as well as data which appears to be relevant
802 805 to the hostOS rather than the guest.
803 806
804 807 For each entry in the nics list returned from query sdc:nics, we
805 808 create a type: physical entry, and extract the interface properties:
806 809 'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'. The remaining
807 810 keys are related to ip configuration. For each ip in the 'ips' list
808 811 we create a subnet entry under 'subnets' pairing the ip to a one in
809 812 the 'gateways' list.
810 813 """
811 814
812 815 valid_keys = {
813 816 'physical': [
814 817 'mac_address',
815 818 'mtu',
816 819 'name',
817 820 'params',
818 821 'subnets',
819 822 'type',
820 823 ],
821 824 'subnet': [
|
↓ open down ↓ |
19 lines elided |
↑ open up ↑ |
822 825 'address',
823 826 'broadcast',
824 827 'dns_nameservers',
825 828 'dns_search',
826 829 'metric',
827 830 'pointopoint',
828 831 'routes',
829 832 'scope',
830 833 'type',
831 834 ],
835 + 'route': [
836 + 'destination',
837 + 'gateway',
838 + ],
832 839 }
833 840
834 841 if dns_servers:
835 842 if not isinstance(dns_servers, (list, tuple)):
836 843 dns_servers = [dns_servers]
837 844 else:
838 845 dns_servers = []
839 846
840 847 if dns_domain:
841 848 if not isinstance(dns_domain, (list, tuple)):
842 849 dns_domain = [dns_domain]
843 850 else:
844 851 dns_domain = []
845 852
846 853 def is_valid_ipv4(addr):
847 854 return '.' in addr
848 855
849 856 def is_valid_ipv6(addr):
850 857 return ':' in addr
851 858
852 859 pgws = {
853 860 'ipv4': {'match': is_valid_ipv4, 'gw': None},
854 861 'ipv6': {'match': is_valid_ipv6, 'gw': None},
855 862 }
856 863
857 864 config = []
858 865 for nic in network_data:
859 866 cfg = dict((k, v) for k, v in nic.items()
860 867 if k in valid_keys['physical'])
861 868 cfg.update({
862 869 'type': 'physical',
863 870 'name': nic['interface']})
864 871 if 'mac' in nic:
865 872 cfg.update({'mac_address': nic['mac']})
866 873
867 874 subnets = []
868 875 for ip in nic.get('ips', []):
869 876 if ip == "dhcp":
870 877 subnet = {'type': 'dhcp4'}
871 878 else:
872 879 subnet = dict((k, v) for k, v in nic.items()
873 880 if k in valid_keys['subnet'])
874 881 subnet.update({
875 882 'type': 'static',
876 883 'address': ip,
877 884 })
878 885
879 886 proto = 'ipv4' if is_valid_ipv4(ip) else 'ipv6'
880 887 # Only use gateways for 'primary' nics
881 888 if 'primary' in nic and nic.get('primary', False):
882 889 # the ips and gateways list may be N to M, here
883 890 # we map the ip index into the gateways list,
884 891 # and handle the case that we could have more ips
885 892 # than gateways. we only consume the first gateway
886 893 if not pgws[proto]['gw']:
|
↓ open down ↓ |
45 lines elided |
↑ open up ↑ |
887 894 gateways = [gw for gw in nic.get('gateways', [])
888 895 if pgws[proto]['match'](gw)]
889 896 if len(gateways):
890 897 pgws[proto]['gw'] = gateways[0]
891 898 subnet.update({'gateway': pgws[proto]['gw']})
892 899
893 900 subnets.append(subnet)
894 901 cfg.update({'subnets': subnets})
895 902 config.append(cfg)
896 903
904 + if routes:
905 + for route in routes:
906 + cfg = dict((k, v) for k, v in route.items()
907 + if k in valid_keys['route'])
908 + # Linux uses the value of 'gateway' to determine automatically if
909 + # the route is a forward/next-hop (non-local IP for gateway) or an
910 + # interface/resolver (local IP for gateway). So we can ignore
911 + # the 'interface' attribute of sdc:routes, because SDC guarantees
912 + # that the gateway is a local IP for "interface=true".
913 + cfg.update({
914 + 'type': 'route',
915 + 'destination': route['destination'],
916 + 'gateway': route['gateway']})
917 + config.append(cfg)
918 +
897 919 if dns_servers:
898 920 config.append(
899 921 {'type': 'nameserver', 'address': dns_servers,
900 922 'search': dns_domain})
901 923
902 924 return {'version': 1, 'config': config}
903 925
904 926
905 927 # Used to match classes to dependencies
906 928 datasources = [
907 929 (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )),
908 930 ]
909 931
910 932
911 933 # Return a list of data sources that match this set of dependencies
912 934 def get_datasource_list(depends):
913 935 return sources.list_from_depends(depends, datasources)
914 936
915 937
916 938 if __name__ == "__main__":
917 939 import sys
918 940 jmc = jmc_client_factory()
919 941 if jmc is None:
920 942 print("Do not appear to be on smartos.")
921 943 sys.exit(1)
922 944 if len(sys.argv) == 1:
923 945 keys = (list(SMARTOS_ATTRIB_JSON.keys()) +
924 946 list(SMARTOS_ATTRIB_MAP.keys()) + ['network_config'])
925 947 else:
|
↓ open down ↓ |
19 lines elided |
↑ open up ↑ |
926 948 keys = sys.argv[1:]
927 949
928 950 def load_key(client, key, data):
929 951 if key in data:
930 952 return data[key]
931 953
932 954 if key in SMARTOS_ATTRIB_JSON:
933 955 keyname = SMARTOS_ATTRIB_JSON[key]
934 956 data[key] = client.get_json(keyname)
935 957 elif key == "network_config":
936 - for depkey in ('network-data', 'dns_servers', 'dns_domain'):
958 + for depkey in ('network-data', 'dns_servers', 'dns_domain', 'routes'):
937 959 load_key(client, depkey, data)
938 960 data[key] = convert_smartos_network_data(
939 961 network_data=data['network-data'],
940 962 dns_servers=data['dns_servers'],
941 - dns_domain=data['dns_domain'])
963 + dns_domain=data['dns_domain'],
964 + routes=data['routes'])
942 965 else:
943 966 if key in SMARTOS_ATTRIB_MAP:
944 967 keyname, strip = SMARTOS_ATTRIB_MAP[key]
945 968 else:
946 969 keyname, strip = (key, False)
947 970 data[key] = client.get(keyname, strip=strip)
948 971
949 972 return data[key]
950 973
951 974 data = {}
952 975 for key in keys:
953 976 load_key(client=jmc, key=key, data=data)
954 977
955 978 print(json.dumps(data, indent=1, sort_keys=True,
956 979 separators=(',', ': ')))
957 980
958 981 # vi: ts=4 expandtab
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX