path: root/bbot/machine.cxx

// file      : bbot/machine.cxx -*- C++ -*-
// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
// license   : TBC; see accompanying LICENSE file

#include <bbot/machine>

#include <unistd.h> // sleep()

#include <sys/un.h>   // sockaddr_un
#include <sys/socket.h>

#include <cstdio>  // snprintf()
#include <cstring> // strcpy()

#include <bbot/agent>
#include <bbot/machine-manifest>

using namespace std;
using namespace butl;

namespace bbot
{
  // Forward TFTP requests (UDP/69) coming from the machine to the specified
  // port.
  //
  // This allows the machine to connect to any "unknown" IP (e.g., link-local
  // 196.254.111.222) port 69 and end up being redirected to out TFTP server.
  //
  static void
  iptables (tracer& t,
            const char* a,
            const string& tap,
            const string& br,
            uint16_t port)
  {
    run (t,
         "sudo",             "iptables",
         "-t",               "nat",
         a,                  "PREROUTING",
         "-p",               "udp",
         "-m",               "udp",
         "-m",               "physdev",
         "-i",               br,
         "--physdev-in",     tap,
         "--dport",          69,
         "-j",               "DNAT",
         "--to-destination", iface_addr (br) + ':' + to_string (port));
  }

  static string
  create_tap (const string& br, uint16_t port)
  {
    tracer trace ("create_tap");

    string t ("tap" + tc_num);

    // First try to delete it in case there is one from a previous run.
    //
    run_exit (trace, "sudo", "ip", "tuntap", "delete", t, "mode", "tap");

    run (trace, "sudo", "ip", "tuntap", "add", t, "mode", "tap", "user", uid);
    run (trace, "sudo", "ip", "link", "set", t, "up");
    run (trace, "sudo", "ip", "link", "set", t, "master", br);

    iptables (trace, "-A", t, br, port); // Add.

    return t;
  }

  static void
  destroy_tap (const string& t, const string& br, uint16_t port)
  {
    tracer trace ("destroy_tap");
    iptables (trace, "-D", t, br, port); // Delete.
    run (trace, "sudo", "ip", "tuntap", "delete", t, "mode", "tap");
  }

  static string
  generate_mac ()
  {
    // The last two bits of the first byte are special: bit 1 indicates a
    // multicast address (which we don't want) while bit 1 -- local assignment
    // (which we do want).
    //
    char r[6 * 2 + 5 + 1];
    snprintf (r, sizeof (r),
              "%02x:%02x:%02x:%02x:%02x:%02x",
              (genrand<uint8_t> () & 0xFE) | 0x02,
              genrand<uint8_t> (),
              genrand<uint8_t> (),
              genrand<uint8_t> (),
              genrand<uint8_t> (),
              genrand<uint8_t> ());
    return r;
  }

  class kvm_machine: public machine
  {
  public:
    kvm_machine (const dir_path&,
                 const machine_manifest&,
                 const optional<string>& mac,
                 const string& br_iface,
                 uint16_t tftp_port);

    virtual bool
    shutdown (size_t& seconds) override;

    virtual void
    forcedown () override;

    virtual void
    suspend () override;

    bool
    wait (size_t& seconds) override;

    using machine::wait;

    virtual void
    print_info (diag_record&) override;

  private:
    void
    monitor_command (const string&);

  private:
    path kvm;     // Hypervisor binary.

    string br;     // Bridge network interface.
    string tap;    // Tap network interface.
    uint16_t port; // TFTP port.

    string vnc;   // QEMU VNC TCP addr:port.
    path monitor; // QEMU monitor UNIX socket.
    process proc;
  };

  kvm_machine::
  kvm_machine (const dir_path& md,
               const machine_manifest& mm,
               const optional<string>& omac,
               const string& br,
               uint16_t port)
      : machine (mm.mac ? *mm.mac : // Fixed mac from machine manifest.
                 omac   ? *omac   : // Generated mac from previous bootstrap.
                 generate_mac ()),
        kvm ("kvm"),
        br (br),
        tap (create_tap (br, port)),
        port (port),
        vnc ("127.0.0.1:" + to_string (5900 + stoul (tc_num))),
        monitor ("/tmp/" + tc_name + "-monitor")
  {
    tracer trace ("kvm_machine");

    if (sizeof (sockaddr_un::sun_path) <= monitor.size ())
      throw invalid_argument ("monitor unix socket path too long");

    // Start the VM.
    //
    // Notes:
    //
    // 1. For now we let qemu calculate sockets/cores/threads from the
    //    total number of CPUs (i.e., threads).
    //
    // 2. echo system_powerdown | socat - UNIX-CONNECT:.../monitor
    //
    proc = run_io_start (
      trace,
      fdnull (),
      2,
      2,
      md, // Run from the machine's directory.
      kvm,
      "-boot", "c", // Boot from disk.
      "-no-reboot", // Exit on VM reboot.
      //
      // Machine.
      //
      "-m",   to_string (ops.ram () / 1024) + "M",
      "-cpu", "host",
      "-smp", ops.cpu (),
      //
      // Network.
      //
      "-device", "virtio-net-pci,netdev=net0,mac=" + mac,
      "-netdev", "tap,id=net0,script=no,ifname=" + tap,
      //
      // Disk.
      //
      "-device", "virtio-scsi-pci,id=scsi",
      "-device", "scsi-hd,drive=disk0",
      "-drive",  "if=none,id=disk0,format=raw,file=disk.img",
      //
      // VNC & monitor.
      //
      "-vnc", "127.0.0.1:" + tc_num, // 5900 + tc_num
      "-monitor", "unix:" + monitor.string () + ",server,nowait");
  }

  // Connect to the QEMU monitor via the UNIX socket and send system_reset.
  // You may be wondering why not system_powerdown? The reason is that while
  // not all OS know how to power-down the machine, pretty much all of them
  // can reboot. So combined with the -no-reboot option above, we get the
  // same result in a more robust way.
  //
  // Note that this setup has one side effect: if the VM decided to reboot,
  // say, during bootstrap, then we will interpret it as a shutdown. Current
  // thinking saying this is good since we don't want our VMs to reboot
  // uncontrollably for security and predictability reasons (e.g., we don't
  // want Windows to decide to install updates -- this stuff should all be
  // disabled during the VM preparation).
  //
  // Actually, this turned out not to be entirely accurate: reset appears to
  // be a "hard reset" while powerdown causes a clean shutdown. So we use
  // powerdown to implement shutdown() and reset/-no-reboot for implement
  // forcedown().
  //
  bool kvm_machine::
  shutdown (size_t& seconds)
  {
    monitor_command ("system_powerdown");

    // Wait for up to the specified number if seconds for the machine to
    // shutdown.
    //
    return wait (seconds);
  }

  void kvm_machine::
  forcedown ()
  {
    monitor_command ("system_reset");
    wait ();
  }

  void kvm_machine::
  suspend ()
  {
    monitor_command ("stop");
  }

  void kvm_machine::
  print_info (diag_record& dr)
  {
    dr << info << "qemu pid: " << proc.id ()
       << info << "qemu vnc: " << vnc
       << info << "qemu monitor: unix:" << monitor;
  }

  bool kvm_machine::
  wait (size_t& sec)
  try
  {
    tracer trace ("kvm_machine::wait");

    bool t;
    for (; !(t = proc.try_wait ()) && sec != 0; --sec)
      sleep (1);

    if (t)
    {
      run_io_finish (trace, proc, kvm);

      destroy_tap (tap, br, port);
      try_rmfile (monitor, true); // QEMU doesn't seem to remove it.
    }

    return t;
  }
  catch (const process_error& e)
  {
    fail << "unable to execute " << kvm << ": " << e << endf;
  }

  void kvm_machine::
  monitor_command (const string& c)
  try
  {
    sockaddr_un addr;
    addr.sun_family = AF_LOCAL;
    strcpy (addr.sun_path, monitor.string ().c_str ()); // Size check in ctor

    auto_fd sock (socket (AF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0));

    if (sock.get () == -1)
      throw_system_error (errno);

    if (connect (sock.get (),
                 reinterpret_cast<sockaddr*> (&addr),
                 sizeof (addr)) == -1)
      throw_system_error (errno);

    // Read until we get something.
    //
    auto readsome = [&sock] ()
    {
      ifdstream ifs (move (sock),
                     fdstream_mode::non_blocking,
                     ostream::badbit);

      char buf[256];
      for (streamsize n (0), m (0);
           n == 0 || m != 0;
           m = ifs.readsome (buf, sizeof (buf) - 1))
      {
        if (m != 0)
        {
          n += m;

          //buf[m] = '\0';
          //text << buf;
        }
      }

      sock = ifs.release ();
    };

    // Read QEMU welcome.
    //
    readsome ();

    // Write our command.
    //
    {
      ofdstream ofs (move (sock), fdstream_mode::blocking);
      ofs << c << endl;
      sock = ofs.release ();
    }

    // Read QEMU reply (may hit eof).
    //
    readsome ();
  }
  catch (const system_error& e)
  {
    fail << "unable to communicate with qemu monitor: " << e;
  }

  unique_ptr<machine>
  start_machine (const dir_path& md,
                 const machine_manifest& mm,
                 const optional<string>& mac,
                 const string& br_iface,
                 uint16_t tftp_port)
  {
    switch (mm.type)
    {
    case machine_type::kvm:
      return make_unique<kvm_machine> (md, mm, mac, br_iface, tftp_port);
    case machine_type::nspawn:
      assert (false); //@@ TODO
    }

    return nullptr;
  }
}