diff options
Diffstat (limited to 'mod/mod-ci-github.cxx')
| -rw-r--r-- | mod/mod-ci-github.cxx | 952 |
1 files changed, 0 insertions, 952 deletions
diff --git a/mod/mod-ci-github.cxx b/mod/mod-ci-github.cxx index 22ba205..179eeba 100644 --- a/mod/mod-ci-github.cxx +++ b/mod/mod-ci-github.cxx @@ -329,77 +329,6 @@ namespace brep } } - // Service data associated with the tenant/check suite. - // - // It is always a top-level JSON object and the first member is always the - // schema version. - // - struct service_data - { - // The data schema version. Note: must be first member in the object. - // - uint64_t version = 1; - - // Check suite-global data. - // - installation_access_token installation_access; - - uint64_t installation_id; - // @@ TODO Rename to repository_node_id. - // - string repository_id; // GitHub-internal opaque repository id. - - string head_sha; - - // Absent state means we were unable to (conclusively) notify GitHub about - // the last state transition (e.g., due to a transient network error). The - // "conclusively" part means that the notification may or may not have - // gone through. Note: node_id can be absent for the same reason. - // - struct check_run - { - string build_id; // Full build id. - optional<string> node_id; // GitHub id. - optional<build_state> state; - - string - state_string () const - { - return state ? to_string (*state) : "null"; - } - }; - vector<check_run> check_runs; - - // Return the check run with the specified build ID or nullptr if not - // found. - // - check_run* - find_check_run (const string& build_id); - - // Construct from JSON. - // - // Throw invalid_argument if the schema version is not supported. - // - explicit - service_data (const string& json); - - service_data (string iat_token, - timestamp iat_expires_at, - uint64_t installation_id, - string repository_id, - string head_sha); - - service_data () = default; - - // Serialize to JSON. - // - string - json () const; - }; - - ostream& - operator<< (ostream&, const service_data::check_run&); - bool ci_github:: handle_check_suite_request (check_suite_event cs) { @@ -453,887 +382,6 @@ namespace brep return true; } - // GraphQL serialization functions. - // - // The GraphQL spec: - // https://spec.graphql.org/ - // - // The GitHub GraphQL API reference: - // https://docs.github.com/en/graphql/reference/ - // - - // Check that a string is a valid GraphQL name. - // - // GraphQL names can contain only alphanumeric characters and underscores - // and cannot begin with a digit (so basically a C identifier). - // - // Return the name or throw invalid_argument if it is invalid. - // - static const string& - gq_name (const string& v) - { - if (v.empty () || digit (v[0])) - throw invalid_argument ("invalid GraphQL name: '" + v + '\''); - - for (char c: v) - { - if (!alnum (c) && c != '_') - { - throw invalid_argument ("invalid character in GraphQL name: '" + c + - '\''); - } - } - - return v; - } - - // Serialize a string to GraphQL. - // - // Return the serialized string or throw invalid_argument if the string is - // invalid. - // - static string - gq_str (const string& v) - { - // GraphQL strings are the same as JSON strings so we use the JSON - // serializer. - // - string b; - json::buffer_serializer s (b); - - try - { - s.value (v); - } - catch (const json::invalid_json_output&) - { - throw invalid_argument ("invalid GraphQL string: '" + v + '\''); - } - - return b; - } - - // Serialize an int to GraphQL. - // -#if 0 - static string - gq_int (uint64_t v) - { - string b; - json::buffer_serializer s (b); - s.value (v); - return b; - } -#endif - - // Serialize a boolean to GraphQL. - // - static inline string - gq_bool (bool v) - { - return v ? "true" : "false"; - } - - // Check that a string is a valid GraphQL enum value. - // - // GraphQL enum values can be any GraphQL name except for `true`, `false`, - // or `null`. - // - // Return the enum value or throw invalid_argument if it is invalid. - // - static const string& - gq_enum (const string& v) - { - if (v == "true" || v == "false" || v == "null") - throw invalid_argument ("invalid GraphQL enum value: '" + v + '\''); - - return gq_name (v); - } - - // Create a check_run name from a build. If the second argument is not - // NULL, return an abbreviated id if possible. - // - static string - check_run_name (const build& b, - const tenant_service_base::build_hints* bh = nullptr) - { - string r; - - if (bh == nullptr || !bh->single_package_version) - { - r += b.package_name.string (); - r += '/'; - r += b.package_version.string (); - r += '/'; - } - - r += b.target_config_name; - r += '/'; - r += b.target.string (); - r += '/'; - - if (bh == nullptr || !bh->single_package_config) - { - r += b.package_config_name; - r += '/'; - } - - r += b.toolchain_name; - r += '-'; - r += b.toolchain_version.string (); - - return r; - } - - // Return the GitHub check run status corresponding to a build_state. - // - static const string& - to_string_gh (build_state st) - { - static const string sts[] {"QUEUED", "IN_PROGRESS", "COMPLETED"}; - - return sts[static_cast<size_t> (st)]; - } - - // Return the build_state corresponding to a GitHub check run status - // string. Throw invalid_argument if the passed status was invalid. - // - static build_state - from_string_gh (const string& s) - { - if (s == "QUEUED") return build_state::queued; - else if (s == "IN_PROGRESS") return build_state::building; - else if (s == "COMPLETED") return build_state::built; - else - throw invalid_argument ("invalid GitHub check run status: '" + s + - '\''); - } - - // Serialize `createCheckRun` mutations for one or more builds to GraphQL. - // - static string - create_check_runs (const string& ri, // Repository ID - const string& hs, // Head SHA - const vector<reference_wrapper<const build>>& bs, - build_state st, - const tenant_service_base::build_hints* bh = nullptr) - { - ostringstream os; - - os << "mutation {" << '\n'; - - // Serialize a `createCheckRun` for each build. - // - for (size_t i (0); i != bs.size (); ++i) - { - const build& b (bs[i]); - - string al ("cr" + to_string (i)); // Field alias. - - // Check run name. - // - string nm (check_run_name (b, bh)); - - os << gq_name (al) << ":createCheckRun(input: {" << '\n' - << " name: " << gq_str (nm) << ',' << '\n' - << " repositoryId: " << gq_str (ri) << ',' << '\n' - << " headSha: " << gq_str (hs) << ',' << '\n' - << " status: " << gq_enum (to_string_gh (st)) << '\n' - << "})" << '\n' - // Specify the selection set (fields to be returned). - // - << "{" << '\n' - << " checkRun {" << '\n' - << " id," << '\n' - << " name," << '\n' - << " status" << '\n' - << " }" << '\n' - << "}" << '\n'; - } - - os << "}" << '\n'; - - return os.str (); - } - - // Serialize an `updateCheckRun` mutation for one build to GraphQL. - // - // @@ TODO Support conclusion, output, etc. - // - static string - update_check_run (const string& ri, // Repository ID - const string& ci, // Check run node_id - build_state st) - { - ostringstream os; - - os << "mutation {" << '\n' - << "cr0:updateCheckRun(input: {" << '\n' - << " checkRunId: " << gq_str (ci) << ',' << '\n' - << " repositoryId: " << gq_str (ri) << ',' << '\n' - << " status: " << gq_enum (to_string_gh (st)) << '\n' - << "})" << '\n' - // Specify the selection set (fields to be returned). - // - << "{" << '\n' - << " checkRun {" << '\n' - << " id," << '\n' - << " name," << '\n' - << " status" << '\n' - << " }" << '\n' - << "}" << '\n' - << "}" << '\n'; - - return os.str (); - } - - // Serialize a GraphQL operation (query/mutation) into a GraphQL request. - // - // This is essentially a JSON object with a "query" string member containing - // the GraphQL operation. For example: - // - // { "query": "mutation { cr0:createCheckRun(... }" } - // - static string - graphql_request (const string& o) - { - string b; - json::buffer_serializer s (b); - - s.begin_object (); - s.member ("query", o); - s.end_object (); - - return b; - } - - [[noreturn]] void - throw_json (json::parser& p, const string& m) - { - throw json::invalid_json_input ( - p.input_name, - p.line (), p.column (), p.position (), - m); - } - - // Parse a JSON-serialized GraphQL response. - // - // Throw runtime_error if the response indicated errors and - // invalid_json_input if the GitHub response contained invalid JSON. - // - // The response format is defined in the GraphQL spec: - // https://spec.graphql.org/October2021/#sec-Response. - // - // Example response: - // - // { - // "data": {...}, - // "errors": {...} - // } - // - // The contents of `data`, including its opening and closing braces, are - // parsed by the `parse_data` function. - // - // @@ TODO errors comes before data in GitHub's responses. - // - static void - parse_graphql_response (json::parser& p, - function<void (json::parser&)> parse_data) - { - using event = json::event; - - // True if the data/errors fields are present. - // - // Although the spec merely recommends that the `errors` field, if - // present, comes before the `data` field, assume it always does because - // letting the client parse data in the presence of field errors - // (unexpected nulls) would not make sense. - // - bool dat (false), err (false); - - p.next_expect (event::begin_object); - - while (p.next_expect (event::name, event::end_object)) - { - if (p.name () == "data") - { - dat = true; - - // Currently we're not handling fields that are null due to field - // errors (see below for details) so don't parse any further. - // - if (err) - break; - - parse_data (p); - } - else if (p.name () == "errors") - { - // Don't stop parsing because the error semantics depends on whether - // or not `data` is present. - // - err = true; // Handled below. - } - else - { - // The spec says the response will never contain any top-level fields - // other than data, errors, and extensions. - // - if (p.name () != "extensions") - { - throw_json (p, - "unexpected top-level GraphQL response field: '" + - p.name () + '\''); - } - - p.next_expect_value_skip (); - } - } - - // If the `errors` field was present in the response, error(s) occurred - // before or during execution of the operation. - // - // If the `data` field was not present the errors are request errors which - // occur before execution and are typically the client's fault. - // - // If the `data` field was also present in the response the errors are - // field errors which occur during execution and are typically the GraphQL - // endpoint's fault, and some fields in `data` that should not be are - // likely to be null. - // - if (err) - { - if (dat) - { - // @@ TODO: Consider parsing partial data? - // - throw runtime_error ("field error(s) received from GraphQL endpoint; " - "incomplete data received"); - } - else - throw runtime_error ("request error(s) received from GraphQL endpoint"); - } - } - - // Parse a response to a check_run GraphQL mutation such as `createCheckRun` - // or `updateCheckRun`. - // - // Example response (only the part we need to parse here): - // - // { - // "cr0": { - // "checkRun": { - // "id": "CR_kwDOLc8CoM8AAAAFQ5GqPg", - // "name": "libb2/0.98.1+2/x86_64-linux-gnu/linux_debian_12-gcc_13.1-O3/default/dev/0.17.0-a.1", - // "status": "QUEUED" - // } - // }, - // "cr1": { - // "checkRun": { - // "id": "CR_kwDOLc8CoM8AAAAFQ5GqhQ", - // "name": "libb2/0.98.1+2/x86_64-linux-gnu/linux_debian_12-gcc_13.1/default/dev/0.17.0-a.1", - // "status": "QUEUED" - // } - // } - // } - // - // @@ TODO Handle response errors properly. - // - static vector<check_run> - parse_check_runs_response (json::parser& p) - { - using event = json::event; - - vector<check_run> r; - - parse_graphql_response ( - p, - [&r] (json::parser& p) - { - p.next_expect (event::begin_object); - - // Parse the "cr0".."crN" members (field aliases). - // - while (p.next_expect (event::name, event::end_object)) - { - // Parse `"crN": { "checkRun":`. - // - if (p.name () != "cr" + to_string (r.size ())) - throw_json (p, "unexpected field alias: '" + p.name () + '\''); - p.next_expect (event::begin_object); - p.next_expect_name ("checkRun"); - - r.emplace_back (p); // Parse the check_run object. - - p.next_expect (event::end_object); // Parse end of crN object. - } - }); - - // Our requests always operate on at least one check run so if there were - // none in the data field something went wrong. - // - if (r.empty ()) - throw_json (p, "data object is empty"); - - return r; - } - - // Send a POST request to the GitHub API endpoint `ep`, parse GitHub's JSON - // response into `rs` (only for 200 codes), and return the HTTP status code. - // - // The endpoint `ep` should not have a leading slash. - // - // Pass additional HTTP headers in `hdrs`. For example: - // - // "HeaderName: header value" - // - // Throw invalid_argument if unable to parse the response headers, - // invalid_json_input (derived from invalid_argument) if unable to parse the - // response body, and system_error in other cases. - // - template <typename T> - static uint16_t - github_post (T& rs, - const string& ep, - const strings& hdrs, - const string& body = "") - { - // Convert the header values to curl header option/value pairs. - // - strings hdr_opts; - - for (const string& h: hdrs) - { - hdr_opts.push_back ("--header"); - hdr_opts.push_back (h); - } - - // Run curl. - // - try - { - // Pass --include to print the HTTP status line (followed by the response - // headers) so that we can get the response status code. - // - // Suppress the --fail option which causes curl to exit with status 22 - // in case of an error HTTP response status code (>= 400) otherwise we - // can't get the status code. - // - // Note that butl::curl also adds --location to make curl follow redirects - // (which is recommended by GitHub). - // - // The API version `2022-11-28` is the only one currently supported. If - // the X-GitHub-Api-Version header is not passed this version will be - // chosen by default. - // - fdpipe errp (fdopen_pipe ()); // stderr pipe. - - curl c (path ("-"), // Read input from curl::out. - path ("-"), // Write response to curl::in. - process::pipe (errp.in.get (), move (errp.out)), - curl::post, - curl::flags::no_fail, - "https://api.github.com/" + ep, - "--no-fail", // Don't fail if response status code >= 400. - "--include", // Output response headers for status code. - "--header", "Accept: application/vnd.github+json", - "--header", "X-GitHub-Api-Version: 2022-11-28", - move (hdr_opts)); - - ifdstream err (move (errp.in)); - - // Parse the HTTP response. - // - uint16_t sc; // Status code. - try - { - // Note: re-open in/out so that they get automatically closed on - // exception. - // - ifdstream in (c.in.release (), fdstream_mode::skip); - ofdstream out (c.out.release ()); - - // Write request body to out. - // - if (!body.empty ()) - out << body; - out.close (); - - sc = curl::read_http_status (in).code; // May throw invalid_argument. - - // Parse the response body if the status code is in the 200 range. - // - if (sc >= 200 && sc < 300) - { - // Use endpoint name as input name (useful to have it propagated - // in exceptions). - // - json::parser p (in, ep /* name */); - rs = T (p); - } - - in.close (); - } - catch (const io_error& e) - { - // If the process exits with non-zero status, assume the IO error is due - // to that and fall through. - // - if (c.wait ()) - { - throw_generic_error ( - e.code ().value (), - (string ("unable to read curl stdout: ") + e.what ()).c_str ()); - } - } - catch (const json::invalid_json_input&) - { - // If the process exits with non-zero status, assume the JSON error is - // due to that and fall through. - // - if (c.wait ()) - throw; - } - - if (!c.wait ()) - { - string et (err.read_text ()); - throw_generic_error (EINVAL, - ("non-zero curl exit status: " + et).c_str ()); - } - - err.close (); - - return sc; - } - catch (const process_error& e) - { - throw_generic_error ( - e.code ().value (), - (string ("unable to execute curl:") + e.what ()).c_str ()); - } - catch (const io_error& e) - { - // Unable to read diagnostics from stderr. - // - throw_generic_error ( - e.code ().value (), - (string ("unable to read curl stderr : ") + e.what ()).c_str ()); - } - } - - // @@ TODO Pass error, trace in same order everywhere. - - // Fetch from GitHub the check run with the specified name (hints-shortened - // build ID). - // - // Return the check run or nullopt if no such check run exists. - // - // In case of error diagnostics will be issued and false returned in second. - // - // Note that the existence of more than one check run with the same name is - // considered an error and reported as such. The API docs imply that there - // can be more than one check run with the same name in a check suite, but - // the observed behavior is that creating a check run destroys the extant - // one, leaving only the new one with different node ID. - // - static pair<optional<gh::check_run>, bool> - fetch_check_run (const string& iat, - const string& check_suite_id, - const string& cr_name, - const basic_mark& error) noexcept - { - try - { - // Example request: - // - // query { - // node(id: "CS_kwDOLc8CoM8AAAAFQPQYEw") { - // ... on CheckSuite { - // checkRuns(last: 100, filterBy: {checkName: "linux_debian_..."}) { - // totalCount, - // edges { - // node { - // id, name, status - // } - // } - // } - // } - // } - // } - // - // This request does the following: - // - // - Look up the check suite by node ID ("direct node lookup"). This - // returns a Node (GraphQL interface). - // - // - Get to the concrete CheckSuite type by using a GraphQL "inline - // fragment" (`... on CheckSuite`). - // - // - Get the check suite's check runs - // - Filter by the sought name - // - Return only two check runs, just enough to be able to tell - // whether there are more than one check runs with this name (which - // is an error). - // - // - Return the id, name, and status fields from the matching check run - // objects. - // - string rq; - { - ostringstream os; - - os << "query {" << '\n'; - - os << "node(id: " << gq_str (check_suite_id) << ") {" << '\n' - << " ... on CheckSuite {" << '\n' - << " checkRuns(last: 2," << '\n' - << " filterBy: {" << '\n' - << "checkName: " << gq_str (cr_name) << '\n' - << " })" << '\n' - // Specify the selection set (fields to be returned). Note that - // edges and node are mandatory. - // - << " {" << '\n' - << " totalCount," << '\n' - << " edges {" << '\n' - << " node {" << '\n' - << " id, name, status" << '\n' - << " }" << '\n' - << " }" << '\n' - << " }" << '\n' - << " }" << '\n' - << "}" << '\n'; - - os << "}" << '\n'; - - rq = os.str (); - } - - // Example response (the part we need to parse here, at least): - // - // { - // "node": { - // "checkRuns": { - // "totalCount": 1, - // "edges": [ - // { - // "node": { - // "id": "CR_kwDOLc8CoM8AAAAFgeoweg", - // "name": "linux_debian_...", - // "status": "IN_PROGRESS" - // } - // } - // ] - // } - // } - // } - // - struct resp - { - optional<check_run> cr; - size_t cr_count = 0; - - resp (json::parser& p) - { - using event = json::event; - - parse_graphql_response (p, [this] (json::parser& p) - { - p.next_expect (event::begin_object); - p.next_expect_member_object ("node"); - p.next_expect_member_object ("checkRuns"); - - cr_count = p.next_expect_member_number<size_t> ("totalCount"); - - p.next_expect_member_array ("edges"); - - for (size_t i (0); i != cr_count; ++i) - { - p.next_expect (event::begin_object); - p.next_expect_name ("node"); - check_run cr (p); - p.next_expect (event::end_object); - - if (i == 0) - this->cr = move (cr); - } - - p.next_expect (event::end_array); // edges - p.next_expect (event::end_object); // checkRuns - p.next_expect (event::end_object); // node - p.next_expect (event::end_object); - }); - } - - resp () = default; - } rs; - - uint16_t sc (github_post (rs, - "graphql", - strings {"Authorization: Bearer " + iat}, - graphql_request (rq))); - - if (sc == 200) - { - if (rs.cr_count <= 1) - return {rs.cr, true}; - else - { - error << "unexpected number of check runs (" << rs.cr_count - << ") in response"; - } - } - else - error << "failed to get check run by name: error HTTP " - << "response status " << sc; - } - catch (const json::invalid_json_input& e) - { - // Note: e.name is the GitHub API endpoint. - // - error << "malformed JSON in response from " << e.name - << ", line: " << e.line << ", column: " << e.column - << ", byte offset: " << e.position << ", error: " << e; - } - catch (const invalid_argument& e) - { - error << "malformed header(s) in response: " << e; - } - catch (const system_error& e) - { - error << "unable to get check run by name (errno=" << e.code () - << "): " << e.what (); - } - catch (const std::exception& e) - { - error << "unable to get check run by name: " << e.what (); - } - - return {nullopt, false}; - } - - // Send a GraphQL mutation request `rq` that operates on one or more check - // runs. Update the check runs in `crs` with the new state and the node ID - // if unset (note: both fields are optionals). Return false and issue - // diagnostics if the request failed. - // - static bool - mutate_check_runs (vector<service_data::check_run>& crs, - const vector<reference_wrapper<const build>>& bs, - const string& iat, - string rq, - build_state st, - const basic_mark& error) noexcept - { - vector<check_run> rcrs; - - try - { - // Response type which parses a GraphQL response containing multiple - // check_run objects. - // - struct resp - { - vector<check_run> check_runs; // Received check runs. - - resp (json::parser& p) : check_runs (parse_check_runs_response (p)) {} - - resp () = default; - } rs; - - uint16_t sc (github_post (rs, - "graphql", // API Endpoint. - strings {"Authorization: Bearer " + iat}, - move (rq))); - - if (sc == 200) - { - rcrs = move (rs.check_runs); - - if (rcrs.size () == bs.size ()) - { - for (size_t i (0); i != rcrs.size (); ++i) - { - // Validate the check run in the response against the build. - // - const check_run& rcr (rcrs[i]); // Received check run. - const build& b (bs[i]); - - build_state rst (from_string_gh (rcr.status)); // Received state. - - if (rst != build_state::built && rst != st) - { - error << "unexpected check_run status: received '" << rcr.status - << "' but expected '" << to_string_gh (st) << '\''; - - return false; // Fail because something is clearly very wrong. - } - else - { - service_data::check_run& cr (crs[i]); - - if (!cr.node_id) - cr.node_id = move (rcr.node_id); - - cr.state = from_string_gh (rcr.status); - } - } - - return true; - } - else - error << "unexpected number of check_run objects in response"; - } - else - error << "failed to update check run: error HTTP response status " - << sc; - } - catch (const json::invalid_json_input& e) - { - // Note: e.name is the GitHub API endpoint. - // - error << "malformed JSON in response from " << e.name << ", line: " - << e.line << ", column: " << e.column << ", byte offset: " - << e.position << ", error: " << e; - } - catch (const invalid_argument& e) - { - error << "malformed header(s) in response: " << e; - } - catch (const system_error& e) - { - error << "unable to mutate check runs (errno=" << e.code () << "): " - << e.what (); - } - catch (const runtime_error& e) // From parse_check_runs_response(). - { - // GitHub response contained error(s) (could be ours or theirs at this - // point). - // - error << "unable to mutate check runs: " << e; - } - - return false; - } - - static bool - mutate_check_run (service_data::check_run& cr, - const vector<reference_wrapper<const build>>& bs, - const string& iat, - string rq, - build_state st, - const basic_mark& error) noexcept - { - vector<service_data::check_run> crs {move (cr)}; - - bool r (mutate_check_runs (crs, bs, iat, move (rq), st, error)); - - cr = move (crs[0]); - - return r; - } - function<optional<string> (const tenant_service&)> ci_github:: build_queued (const tenant_service& ts, const vector<build>& builds, |