// file : libbbot/manifest.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file #include #include #include #include #include // isxdigit() #include #include #include // size_t #include // move() #include // uint64_t #include // invalid_argument #include #include #include // digit() #include #include #include #include using namespace std; using namespace butl; using namespace bpkg; namespace bbot { using parser = manifest_parser; using parsing = manifest_parsing; using serializer = manifest_serializer; using serialization = manifest_serialization; using name_value = manifest_name_value; using butl::optional; // result_status // string to_string (result_status s) { switch (s) { case result_status::success: return "success"; case result_status::warning: return "warning"; case result_status::error: return "error"; case result_status::abort: return "abort"; case result_status::abnormal: return "abnormal"; } assert (false); return string (); } result_status to_result_status (const string& s) { if (s == "success") return result_status::success; else if (s == "warning") return result_status::warning; else if (s == "error") return result_status::error; else if (s == "abort") return result_status::abort; else if (s == "abnormal") return result_status::abnormal; else throw invalid_argument ("invalid result status '" + s + "'"); } // Utility functions // inline static bool valid_sha256 (const string& s) noexcept { if (s.size () != 64) return false; for (const auto& c: s) { if ((c < 'a' || c > 'f' ) && !digit (c)) return false; } return true; } inline static bool valid_fingerprint (const string& f) noexcept { size_t n (f.size ()); if (n != 32 * 3 - 1) return false; for (size_t i (0); i < n; ++i) { char c (f[i]); if ((i + 1) % 3 == 0) { if (c != ':') return false; } else if (!isxdigit (c)) return false; } return true; } // machine_header_manifest // machine_header_manifest:: machine_header_manifest (parser& p, manifest_unknown_mode m) : machine_header_manifest (p, p.next (), m) { // Make sure this is the end. // name_value nv (p.next ()); if (!nv.empty ()) throw parsing (p.name (), nv.name_line, nv.name_column, "single machine header manifest expected"); } machine_header_manifest:: machine_header_manifest (parser& p, name_value nv, manifest_unknown_mode m, name_value* e) { auto bad_name = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.name_line, nv.name_column, d); }; auto bad_value = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.value_line, nv.value_column, d); }; // Make sure this is the start and we support the version. // if (!nv.name.empty ()) bad_name ("start of machine header manifest expected"); if (nv.value != "1") bad_value ("unsupported format version"); for (nv = p.next (); !nv.empty (); nv = p.next ()) { string& n (nv.name); string& v (nv.value); if (n == "id") { if (!id.empty ()) bad_name ("machine id redefinition"); if (v.empty ()) bad_value ("empty machine id"); id = move (v); } else if (n == "name") { if (!name.empty ()) bad_name ("machine name redefinition"); if (v.empty ()) bad_value ("empty machine name"); name = move (v); } else if (n == "summary") { if (!summary.empty ()) bad_name ("machine summary redefinition"); if (v.empty ()) bad_value ("empty machine summary"); summary = move (v); } else { switch (m) { case manifest_unknown_mode::skip: continue; case manifest_unknown_mode::fail: bad_name ("unknown name '" + n + "' in machine header manifest"); case manifest_unknown_mode::stop: break; // Bail out from the loop. } break; } } // Verify all non-optional values were specified. // if (id.empty ()) bad_value ("no machine id specified"); if (name.empty ()) bad_value ("no machine name specified"); if (summary.empty ()) bad_value ("no machine summary specified"); if (e != nullptr) *e = move (nv); } void machine_header_manifest:: serialize (serializer& s, bool end_of_manifest) const { // @@ Should we check that all non-optional values are specified and all // values are valid? // s.next ("", "1"); // Start of manifest. s.next ("id", id); s.next ("name", name); s.next ("summary", summary); if (end_of_manifest) s.next ("", ""); // End of manifest. } // task_request_manifest // task_request_manifest:: task_request_manifest (parser& p, bool iu) { name_value nv (p.next ()); auto bad_name = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.name_line, nv.name_column, d); }; auto bad_value = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.value_line, nv.value_column, d); }; // Make sure this is the start and we support the version. // if (!nv.name.empty ()) bad_name ("start of task request manifest expected"); if (nv.value != "1") bad_value ("unsupported format version"); // Parse the task request manifest. // for (nv = p.next (); !nv.empty (); nv = p.next ()) { string& n (nv.name); string& v (nv.value); if (n == "agent") { if (!agent.empty ()) bad_name ("task request agent redefinition"); if (v.empty ()) bad_value ("empty task request agent"); agent = move (v); } else if (n == "toolchain-name") { if (!toolchain_name.empty ()) bad_name ("task request toolchain name redefinition"); if (v.empty ()) bad_value ("empty task request toolchain name"); toolchain_name = move (v); } else if (n == "toolchain-version") { if (!toolchain_version.empty ()) bad_name ("task request toolchain version redefinition"); try { toolchain_version = standard_version (v); } catch (const invalid_argument& e) { bad_value (string ("invalid task request toolchain version: ") + e.what ()); } } else if (n == "fingerprint") { if (fingerprint) bad_name ("task request fingerprint redefinition"); if (!valid_sha256 (v)) bad_value ("invalid task request fingerprint"); fingerprint = move (v); } else if (!iu) bad_name ("unknown name '" + n + "' in task request manifest"); } // Verify all non-optional values were specified. // if (agent.empty ()) bad_value ("no task request agent specified"); if (toolchain_name.empty ()) bad_value ("no task request toolchain name specified"); if (toolchain_version.empty ()) bad_value ("no task request toolchain version specified"); // Parse machine header manifests. // for (nv = p.next (); !nv.empty (); nv = p.next ()) machines.emplace_back ( machine_header_manifest (p, nv, iu ? manifest_unknown_mode::skip : manifest_unknown_mode::fail)); if (machines.empty ()) bad_value ("no task request machines specified"); } void task_request_manifest:: serialize (serializer& s) const { // @@ Should we check that all non-optional values are specified and all // values are valid? // s.next ("", "1"); // Start of manifest. s.next ("agent", agent); s.next ("toolchain-name", toolchain_name); s.next ("toolchain-version", toolchain_version.string ()); if (fingerprint) s.next ("fingerprint", *fingerprint); s.next ("", ""); // End of manifest. for (const machine_header_manifest& m: machines) m.serialize (s); s.next ("", ""); // End of stream. } // task_manifest // task_manifest:: task_manifest (parser& p, bool iu) : task_manifest (p, p.next (), iu) { // Make sure this is the end. // name_value nv (p.next ()); if (!nv.empty ()) throw parsing (p.name (), nv.name_line, nv.name_column, "single task manifest expected"); } task_manifest:: task_manifest (parser& p, name_value nv, bool iu) { auto bad_name = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.name_line, nv.name_column, d); }; // Offsets are used to tie an error to the specific position inside a // manifest value (possibly a multiline one). // auto bad_value = [&p, &nv] ( const string& d, uint64_t column_offset = 0, uint64_t line_offset = 0) { throw parsing (p.name (), nv.value_line + line_offset, (line_offset == 0 ? nv.value_column : 1) + column_offset, d); }; // Make sure this is the start and we support the version. // if (!nv.name.empty ()) bad_name ("start of task manifest expected"); if (nv.value != "1") bad_value ("unsupported format version"); // Parse value represented as a whitespace-separated list of quoted // strings (quoting is validated and preserved) and validate each string // with the function specified. // auto parse_tab = [&bad_value] ( const string& value, const function& check, const string& what) -> strings { strings r; // Note that when reporting errors we combine the manifest value // position with the respective field and error positions. // try { istringstream is (value); tab_parser parser (is, ""); // Here we naturally support multiline value manifest. // tab_fields tl; while (!(tl = parser.next ()).empty ()) { for (auto& tf: tl) { try { check (tf.value); } catch (const invalid_argument& e) { bad_value (string ("invalid task ") + what + ": " + e.what (), tf.column - 1, tl.line - 1); } r.emplace_back (move (tf.value)); } } } catch (const tab_parsing& e) { bad_value ("invalid task " + what + ": " + e.description, e.column - 1, e.line - 1); } return r; }; // Parse the task manifest. // // The repository type value can go after the repository URL value. So we // need to postpone creating the repository location until we went though // all other values. // optional repo_url; optional repo_type; for (nv = p.next (); !nv.empty (); nv = p.next ()) { string& n (nv.name); string& v (nv.value); if (n == "name") { if (!name.empty ()) bad_name ("task package name redefinition"); try { name = package_name (move (v)); } catch (const invalid_argument& e) { bad_value (string ("invalid task package name: ") + e.what ()); } } else if (n == "version") { if (!version.empty ()) bad_name ("task package version redefinition"); try { version = bpkg::version (move (v)); } catch (const invalid_argument& e) { bad_value (string ("invalid task package version: ") + e.what ()); } // Versions like 1.2.3- are forbidden in manifest as intended to be // used for version constrains rather than actual releases. // if (version.release && version.release->empty ()) bad_value ("invalid task package version release"); } else if (n == "repository-url") { if (repo_url) bad_name ("task repository URL redefinition"); if (v.empty ()) bad_value ("empty task repository URL"); repo_url = move (nv); } else if (n == "repository-type") { if (repo_type) bad_name ("task repository type redefinition"); try { repo_type = to_repository_type (v); } catch (const invalid_argument&) { bad_value ("invalid task repository type '" + v + "'"); } } else if (n == "trust") { if (v != "yes" && !valid_fingerprint (v)) bad_value ("invalid repository certificate fingerprint"); trust.emplace_back (move (v)); } else if (n == "machine") { if (!machine.empty ()) bad_name ("task machine redefinition"); if (v.empty ()) bad_value ("empty task machine"); machine = move (v); } else if (n == "target") { if (!target.empty ()) bad_name ("task target redefinition"); try { target = target_triplet (v); } catch (const invalid_argument& e) { bad_value (string ("invalid task target: ") + e.what ()); } } else if (n == "environment") { if (environment) bad_name ("task environment redefinition"); if (v.empty ()) bad_value ("empty task environment"); environment = move (v); } else if (n == "config") { if (!config.empty ()) bad_name ("task configuration redefinition"); config = parse_tab (v, [](const string&){}, "configuration"); if (config.empty ()) bad_value ("empty task configuration"); } else if (n == "warning-regex") { if (!warning_regex.empty ()) bad_name ("task warning regex redefinition"); warning_regex = parse_tab (v, validate_regex, "warning regex"); if (warning_regex.empty ()) bad_value ("empty task warning regex"); } else if (!iu) bad_name ("unknown name '" + n + "' in task manifest"); } // Verify all non-optional values were specified. // if (name.empty ()) bad_value ("no task package name specified"); if (version.empty ()) bad_value ("no task package version specified"); if (!repo_url) bad_value ("no task repository URL specified"); if (machine.empty ()) bad_value ("no task machine specified"); if (target.empty ()) bad_value ("no task target specified"); // Create the repository location. // try { // Create remote/absolute repository location (will throw // invalid_argument for relative location). // repository = repository_location (repo_url->value, repo_type); } catch (const invalid_argument& e) { nv = move (*repo_url); // Restore as bad_value() uses its line/column. bad_value (string ("invalid task repository URL: ") + e.what ()); } } void task_manifest:: serialize (serializer& s) const { // @@ Should we check that all non-optional values are specified and all // values are valid? // s.next ("", "1"); // Start of manifest. auto bad_value ([&s](const string& d) { throw serialization (s.name (), d);}); if (name.empty ()) bad_value ("empty task package name"); s.next ("name", name.string ()); s.next ("version", version.string ()); // Note that the repository location is assumed to be remote or absolute, // and so the URL schema encapsulates the repository type if it is // unguessable otherwise. Thus we don't serialize the repository-type // manifest value. // s.next ("repository-url", repository.string ()); for (const auto& v: trust) s.next ("trust", v); s.next ("machine", machine); s.next ("target", target.string ()); if (environment) s.next ("environment", *environment); // Serialize an optional value of the strings type as a space-separated // string list. // auto serialize_list = [&s] (const char* name, const strings& value) { if (!value.empty ()) { string v; for (auto b (value.cbegin ()), i (b), e (value.cend ()); i != e; ++i) { if (i != b) v += ' '; v += *i; } s.next (name, v); } }; serialize_list ("config", config); serialize_list ("warning-regex", warning_regex); s.next ("", ""); // End of manifest. } strings task_manifest:: unquoted_config () const { return string_parser::unquote (config); } strings task_manifest:: unquoted_warning_regex () const { return string_parser::unquote (warning_regex); } void task_manifest:: validate_regex (const string& s) { try { regex re (string_parser::unquote (s)); } catch (const regex_error& e) { // Print regex_error description if meaningful (no space). // ostringstream os; os << "invalid regex" << e; throw invalid_argument (os.str ()); } } // task_response_manifest // task_response_manifest:: task_response_manifest (parser& p, bool iu) { name_value nv (p.next ()); auto bad_name = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.name_line, nv.name_column, d); }; auto bad_value = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.value_line, nv.value_column, d); }; // Make sure this is the start and we support the version. // if (!nv.name.empty ()) bad_name ("start of task response manifest expected"); if (nv.value != "1") bad_value ("unsupported format version"); // Parse the task response manifest. // // Note that we need to distinguish an empty and absent session. // optional sess; for (nv = p.next (); !nv.empty (); nv = p.next ()) { string& n (nv.name); string& v (nv.value); if (n == "session") { if (sess) bad_name ("task response session redefinition"); sess = move (v); } else if (n == "challenge") { if (challenge) bad_name ("task response challenge redefinition"); if (v.size () != 64) bad_value ("invalid task response challenge"); challenge = move (v); } else if (n == "result-url") { if (result_url) bad_name ("task response result url redefinition"); if (v.empty ()) bad_value ("empty task response result url"); result_url = move (v); } else if (!iu) bad_name ("unknown name '" + n + "' in task response manifest"); } // Verify all non-optional values were specified, and all values are // expected. // if (!sess) bad_value ("no task response session specified"); session = move (*sess); // If session is not empty then the challenge may, and the result url // must, be present, otherwise they shouldn't. // if (!session.empty ()) { if (!result_url) bad_value ("no task response result url specified"); } else { if (challenge) bad_value ("unexpected task response challenge"); if (result_url) bad_value ("unexpected task response result url"); } // If session is not empty then the task manifest must follow, otherwise it // shouldn't. // nv = p.next (); if (!session.empty ()) { if (nv.empty ()) bad_value ("task manifest expected"); task = task_manifest (p, nv, iu); nv = p.next (); } // Make sure this is the end. // if (!nv.empty ()) throw parsing (p.name (), nv.name_line, nv.name_column, "single task response manifest expected"); } void task_response_manifest:: serialize (serializer& s) const { // @@ Should we check that all non-optional values are specified and all // values are valid? // s.next ("", "1"); // Start of manifest. s.next ("session", session); if (challenge) s.next ("challenge", *challenge); if (result_url) s.next ("result-url", *result_url); s.next ("", ""); // End of manifest. if (task) task->serialize (s); s.next ("", ""); // End of stream. } // result_manifest // result_manifest:: result_manifest (parser& p, bool iu) : result_manifest (p, p.next (), iu) { // Make sure this is the end. // name_value nv (p.next ()); if (!nv.empty ()) throw parsing (p.name (), nv.name_line, nv.name_column, "single result manifest expected"); } result_manifest:: result_manifest (parser& p, name_value nv, bool iu) { auto bad_name = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.name_line, nv.name_column, d); }; auto bad_value = [&p, &nv] (const string& d, size_t offset = 0) { throw parsing (p.name (), nv.value_line, nv.value_column + offset, d); }; auto result_stat = [&bad_value] (const string& v, const string& what) -> result_status { try { return to_result_status (v); } catch (const invalid_argument&) { bad_value ("invalid " + what); } // Can't be here. Would be redundant if it were possible to declare // lambda with the [[noreturn]] attribute. Note that GCC (non-portably) // supports that. // return result_status::abnormal; }; // Make sure this is the start and we support the version. // if (!nv.name.empty ()) bad_name ("start of result manifest expected"); if (nv.value != "1") bad_value ("unsupported format version"); // Parse the result manifest. // optional stat; // Number of parsed *-log values. Also denotes the next expected log type. // size_t nlog (0); for (nv = p.next (); !nv.empty (); nv = p.next ()) { string& n (nv.name); string& v (nv.value); if (n == "name") { if (!name.empty ()) bad_name ("result package name redefinition"); try { name = package_name (move (v)); } catch (const invalid_argument& e) { bad_value (string ("invalid result package name: ") + e.what ()); } } else if (n == "version") { if (!version.empty ()) bad_name ("result package version redefinition"); try { version = bpkg::version (move (v)); } catch (const invalid_argument& e) { bad_value (string ("invalid result package version: ") + e.what ()); } // Versions like 1.2.3- are forbidden in manifest as intended to be // used for version constrains rather than actual releases. // if (version.release && version.release->empty ()) bad_value ("invalid result package version release"); } else if (n == "status") { if (stat) bad_name ("result status redefinition"); stat = result_stat (v, "result status"); } else { size_t nn (n.size ()); // Name length. // Note: returns false if nothing preceeds a suffix. // auto suffix = [&n, nn] (const char* s, size_t sn) -> bool { return nn > sn && n.compare (nn - sn, sn, s) == 0; }; size_t sn; if (suffix ("-status", sn = 7)) { if (!stat) bad_name ("result status must appear first"); if (nlog > 0) // Some logs have already been parsed. bad_name (n + " after operations logs"); string op (n, 0, nn - sn); // Make sure the operation result status is not redefined. // for (const auto& r: results) { if (r.operation == op) bad_name ("result " + n + " redefinition"); } // Add the operation result (log will come later). // results.push_back ({move (op), result_stat (v, n), string ()}); } else if (suffix ("-log", sn = 4)) { string op (n, 0, nn - sn); // Check that specifically this operation log is expected. // if (nlog >= results.size ()) bad_name ("unexpected " + n); if (results[nlog].operation != op) bad_name (results[nlog].operation + "-log is expected"); // Save operation log. // results[nlog++].log = move (v); } else if (!iu) bad_name ("unknown name '" + n + "' in result manifest"); } } // Verify all non-optional values were specified. // if (name.empty ()) bad_value ("no result package name specified"); if (version.empty ()) bad_value ("no result package version specified"); if (!stat) bad_value ("no result status specified"); // @@ Checking that the result status is consistent with operations // statuses is a bit hairy, so let's postpone for now. // status = move (*stat); // Check that we have log for every operation result status. // if (nlog < results.size ()) bad_name ("no result " + results[nlog].operation + "-log specified"); } void result_manifest:: serialize (serializer& s) const { // @@ Should we check that all non-optional values are specified and all // values are valid? // s.next ("", "1"); // Start of manifest. auto bad_value ([&s](const string& d) { throw serialization (s.name (), d);}); if (name.empty ()) bad_value ("empty result package name"); s.next ("name", name.string ()); s.next ("version", version.string ()); s.next ("status", to_string (status)); // Serialize *-status values. // for (const auto& r: results) s.next (r.operation + "-status", to_string (r.status)); // Serialize *-log values. // for (const auto& r: results) s.next (r.operation + "-log", r.log); s.next ("", ""); // End of manifest. } // result_request_manifest // result_request_manifest:: result_request_manifest (parser& p, bool iu) { name_value nv (p.next ()); auto bad_name = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.name_line, nv.name_column, d); }; auto bad_value = [&p, &nv] (const string& d) { throw parsing (p.name (), nv.value_line, nv.value_column, d); }; // Make sure this is the start and we support the version. // if (!nv.name.empty ()) bad_name ("start of result request manifest expected"); if (nv.value != "1") bad_value ("unsupported format version"); // Parse the result request manifest. // for (nv = p.next (); !nv.empty (); nv = p.next ()) { string& n (nv.name); string& v (nv.value); if (n == "session") { if (!session.empty ()) bad_name ("result request session redefinition"); if (v.empty ()) bad_value ("empty result request session"); session = move (v); } else if (n == "challenge") { if (challenge) bad_name ("result request challenge redefinition"); if (v.empty ()) bad_value ("empty result request challenge"); try { challenge = base64_decode (v); } catch (const invalid_argument&) { bad_value ("invalid result request challenge"); } } else if (!iu) bad_name ("unknown name '" + n + "' in result request manifest"); } // Verify all non-optional values were specified. // if (session.empty ()) bad_value ("no result request session specified"); nv = p.next (); if (nv.empty ()) bad_value ("result manifest expected"); result = result_manifest (p, nv, iu); // Make sure this is the end. // nv = p.next (); if (!nv.empty ()) throw parsing (p.name (), nv.name_line, nv.name_column, "single result request manifest expected"); } void result_request_manifest:: serialize (serializer& s) const { // @@ Should we check that all non-optional values are specified and all // values are valid? // s.next ("", "1"); // Start of manifest. s.next ("session", session); if (challenge) s.next ("challenge", base64_encode (*challenge)); s.next ("", ""); // End of manifest. result.serialize (s); s.next ("", ""); // End of stream. } }