diff options
| -rw-r--r-- | INSTALL-DEV | 4 | ||||
| -rw-r--r-- | INSTALL-GITHUB-DEV | 90 | ||||
| -rw-r--r-- | mod/mod-ci-github.cxx | 392 | ||||
| -rw-r--r-- | mod/mod-ci-github.hxx | 43 | ||||
| -rw-r--r-- | mod/mod-repository-root.cxx | 18 | ||||
| -rw-r--r-- | mod/mod-repository-root.hxx | 2 |
6 files changed, 546 insertions, 3 deletions
diff --git a/INSTALL-DEV b/INSTALL-DEV index 8ebc5a3..f023962 100644 --- a/INSTALL-DEV +++ b/INSTALL-DEV @@ -145,9 +145,9 @@ $ psql -d brep_build -c 'SELECT DISTINCT name FROM build_package' 3. Setup Apache2 Module -Here we assume Apache2 is installed and you have an appropriate VirtualServer +Here we assume Apache2 is installed and you have an appropriate VirtualHost ready (the one for the default site is usually a good candidate). Open the -corresponding Apache2 .conf file and add the following inside VirtualServer, +corresponding Apache2 .conf file and add the following inside VirtualHost, replacing <BREP-OUT-ROOT> and <BREP-SRC-ROOT> with the actual absolute paths (if you built brep in the source tree, then the two would be the same), and replacing <user> with your login. diff --git a/INSTALL-GITHUB-DEV b/INSTALL-GITHUB-DEV new file mode 100644 index 0000000..45f4b9b --- /dev/null +++ b/INSTALL-GITHUB-DEV @@ -0,0 +1,90 @@ +This document explains how to get GitHub webhooks (a notification that an +event such as a push has occurred on a repository) delivered to a +locally-running instance of brep (currently to initiate a CI job). + +0. Overview of the brep GitHub CI integration + +First we register our GitHub CI brep instance as a GitHub app. This GitHub app +essentially consists of an app name, the URL we want webhooks to be delivered +to, permissions required on users' repositories, event subscriptions, and +various authentication-related settings. + +Once registered, GitHub users can install this GitHub app on their user's or +organization's accounts, optionally restricting its access to specific +repositories. + +Once installed on a repository, GitHub will send webhook requests to our app's +webhook URL when, for example, code is pushed or a pull request is created +(the specifics depending on the events our app is subscribed to). + +For development we need these webhooks delivered to our locally-running brep +instance. This is achieved by setting the GitHub app's webhook URL to that of +the webhook proxy smee.io (as recommended by GitHub) and connecting it to our +local brep instance via the locally-run smee client (a Node application). + +1. Follow the instructions in INSTALL-DEV to get brep set up. + +2. Register the GitHub app + +GitHub doc: Registering a GitHub App (note that that doc is a bit out of date) +https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app + +Skip the steps marked "optional" and leave authorization-related settings at +their defaults. + +@@ TODO Update authentication-related info once better understood. + +At this stage the only settings important to us are: + +- App name +- Webhook URL (updated later -- leave webhooks deactivated for now) +- Repository permissions + - Checks: RW + - Pull requests: RO + - Contents: RO + - Metadata: RO +- Subscribed events + - Check suite + - Check run + - Pull request + +3. Install the GitHub app + +GitHub doc: Installing your own GitHub App +https://docs.github.com/en/apps/using-github-apps/installing-your-own-github-app + +It would probably make sense to install it to your own user account and +restrict its access to a test repository. + +4. Forward GitHub webhooks to a local brep instance + +Go to https://smee.io/ and start a new channel. Note the webhook proxy URL, +which will look something like + + https://smee.io/7stvNqVgyQRlIhbY + +Set the GitHub app's webhook URL to this proxy URL. + +Install the smee client: + + $ npm install --global smee-client + +Start brep. + +Start the smee client, passing the webhook proxy URL with --url and the brep +GitHub CI endpoint's URL with --target: + + $ smee --url https://smee.io/7stvNqVgyQRlIhbY \ + --target http://127.0.0.1/pkg?ci-github + +Trigger a webhook delivery from GitHub by pushing a commit to a repository the +GitHub app is installed in. You should see the webhook delivery on the smee.io +channel page and the smee client will also print something to terminal. + +Any webhook delivery can be redelivered by clicking a button on the smee.io +channel page (or the app's advanced settings page on GitHub) so no need to +repeatedly push to the repository. + +You can also see the HTTP headers and JSON payload of delivered webhooks on +both the GitHub app's advanced settings page and the smee.io channel page, but +smee.io's presentation is much better. (There's also wireshark of course.) diff --git a/mod/mod-ci-github.cxx b/mod/mod-ci-github.cxx new file mode 100644 index 0000000..0a28b63 --- /dev/null +++ b/mod/mod-ci-github.cxx @@ -0,0 +1,392 @@ +// file : mod/mod-ci-github.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include <mod/mod-ci-github.hxx> + +#include <libbutl/json/parser.hxx> + +#include <mod/module-options.hxx> + +#include <iostream> + +// @@ TODO +// +// Building CI checks with a GitHub App +// https://docs.github.com/en/apps/creating-github-apps/writing-code-for-a-github-app/building-ci-checks-with-a-github-app +// + +// @@ TODO Best practices +// +// Webhooks: +// https://docs.github.com/en/webhooks/using-webhooks/best-practices-for-using-webhooks +// +// REST API: +// https://docs.github.com/en/rest/using-the-rest-api/best-practices-for-using-the-rest-api?apiVersion=2022-11-28 +// +// Creating an App: +// https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/best-practices-for-creating-a-github-app +// +// Use a webhook secret to ensure request is coming from Github. HMAC: +// https://en.wikipedia.org/wiki/HMAC#Definition. A suitable implementation +// is provided by OpenSSL. + +// @@ Authenticating to use the API +// +// There are three types of authentication: +// +// 1) Authenticating as an app. Used to access parts of the API concerning +// the app itself such as getting the list of installations. (Need to +// authenticate as an app as part of authenticating as an app +// installation.) +// +// 2) Authenticating as an app installation (on a user or organisation +// account). Used to access resources belonging to the user/repository +// or organisation the app is installed in. +// +// 3) Authenticating as a user. Used to perform actions as the user. +// +// We need to authenticate as an app installation (2). +// +// How to authenticate as an app installation +// +// Reference: +// https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation +// +// The final authentication token we need is an installation access token +// (IAT), valid for one hour, which we will pass in the `Authentication` +// header of our Github API requests: +// +// Authorization: Bearer <INSTALLATION_ACCESS_TOKEN> +// +// To generate an IAT: +// +// - Generate a JSON Web Token (JWT) +// +// The inputs are (one of) the application's private key(s) and the +// application ID, which goes into the "issuer" JWT field. Also the +// token's issued time and expiration time (10 minutes max). +// +// The best library for us seems to be libjwt at +// https://github.com/benmcollins/libjwt which is also widely packaged +// (most Linux distros and Homebrew). +// +// Doing it ourselves: +// +// Github requires the RS256 algorithm, which is RSA signing using +// SHA256. +// +// The message consists of a base64url-encoded JSON header and +// payload. (base64url differs from base64 by having different 62nd and +// 63rd alphabet characters (- and _ instead of ~ and . to make it +// filesystem-safe) and no padding because the padding character '=' is +// unsafe in URIs.) +// +// Header: +// +// { +// "alg": "RS256", +// "typ": "JWT" +// } +// +// Payload: +// +// { +// "iat": 1234567, +// "exp": 1234577, +// "iss": "<APP_ID>" +// } +// +// Where: +// iat := Issued At (NumericDate) +// exp := Expires At (NumericDate) +// iss := Issuer +// +// Signature: +// +// RSA_SHA256(PKEY, base64url($header) + "." + base64url($payload)) +// +// JWT: +// +// base64url($header) + "." + +// base64url($payload) + "." + +// base64url($signature) +// +// - Get the installation ID. This will be included in the webhook request +// in our case +// +// - Send a POST to /app/installations/<INSTALLATION_ID>/access_tokens +// which includes the JWT (`Authorization: Bearer <JWT>`). The response +// will include the IAT. Can pass the name of the repository included in +// the webhook request to restrict access, otherwise we get access to all +// repos covered by the installation if installed on an organisation for +// example. + +using namespace std; +using namespace butl; +using namespace web; +using namespace brep::cli; + +brep::ci_github::ci_github (const ci_github& r) + : handler (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::ci_github:: +init (scanner& s) +{ + options_ = make_shared<options::ci> ( + s, unknown_mode::fail, unknown_mode::fail); +} + +// The "check_suite" object within a check_quite webhook request. +// +struct check_suite +{ + uint64_t id; + string head_branch; + string head_sha; + string before; + string after; + + explicit + check_suite (json::parser&); + + check_suite () = default; +}; + +struct repository +{ + string name; + string full_name; + string default_branch; + + explicit repository (json::parser&); + + repository () = default; +}; + +struct check_suite_event +{ + string action; + ::check_suite check_suite; + ::repository repository; + + explicit check_suite_event (json::parser&); + + check_suite_event () = default; +}; + +static ostream& +operator<< (ostream&, const check_suite&); + +static ostream& +operator<< (ostream&, const repository&); + +static ostream& +operator<< (ostream&, const check_suite_event&); + +bool brep::ci_github:: +handle (request& rq, response& rs) +{ + using namespace bpkg; + + HANDLER_DIAG; + + // @@ TODO + if (false) + throw invalid_request (404, "CI request submission disabled"); + + // Process headers. + // + string event; + { + bool content_type (false); + + for (const name_value& h: rq.headers ()) + { + if (icasecmp (h.name, "x-github-delivery") == 0) + { + // @@ TODO Check that delivery UUID has not been received before + // (replay attack). + } + else if (icasecmp (h.name, "content-type") == 0) + { + if (!h.value) + throw invalid_request (400, "missing content-type value"); + + if (icasecmp (*h.value, "application/json") != 0) + { + throw invalid_request (400, + "invalid content-type value: '" + *h.value + + '\''); + } + + content_type = true; + } + else if (icasecmp (h.name, "x-github-event") == 0) + { + if (!h.value) + throw invalid_request (400, "missing x-github-event value"); + + event = *h.value; + } + } + + if (!content_type) + throw invalid_request (400, "missing content-type header"); + + if (event.empty ()) + throw invalid_request (400, "missing x-github-event header"); + } + + // There is an event (specified in the x-github-event header) and each event + // contains a bunch of actions (specified in the JSON request body). + // + // Note: "GitHub continues to add new event types and new actions to + // existing event types." As a result we ignore known actions that we are + // not interested in and log and ignore unknown actions. The thinking here + // is that we want be "notified" of new actions at which point we can decide + // whether to ignore them or to handle. + // + try + { + if (event == "check_suite") + { + json::parser p (rq.content (64 * 1024), "check_suite webhook"); + + check_suite_event cs (p); + + // @@ TODO: log and ignore unknown. + // + if (cs.action == "requested") + { + } + else if (cs.action == "rerequested") + { + // Someone manually requested to re-run the check runs in this check + // suite. + } + else if (cs.action == "completed") + { + // GitHub thinks that "all the check runs in this check suite have + // completed and a conclusion is available". Looks like this one we + // ignore? + } + else + throw invalid_request (400, "unsupported action: " + cs.action); + + cout << "<check_suite webhook>" << endl << cs << endl; + + return true; + } + else if (event == "pull_request") + { + throw invalid_request (501, "pull request events not implemented yet"); + } + else + throw invalid_request (400, "unexpected event: '" + event + "'"); + } + catch (const json::invalid_json_input& e) + { + // @@ TODO: should we write more detailed diagnostics to log? Maybe we + // should do this for all unsuccessful calls to respond(). + // + // @@ TMP These exceptions end up in the apache error log. + // + throw invalid_request (400, "malformed JSON in request body"); + } +} + +using event = json::event; + +// check_suite +// +check_suite::check_suite (json::parser& p) +{ + p.next_expect (event::begin_object); + + // Skip unknown/uninteresting members. + // + while (p.next_expect (event::name, event::end_object)) + { + const string& n (p.name ()); + + if (n == "id") id = p.next_expect_number<uint64_t> (); + else if (n == "head_branch") head_branch = p.next_expect_string (); + else if (n == "head_sha") head_sha = p.next_expect_string (); + else if (n == "before") before = p.next_expect_string (); + else if (n == "after") after = p.next_expect_string (); + else p.next_expect_value_skip (); + } +} + +static ostream& +operator<< (ostream& os, const check_suite& cs) +{ + os << "id: " << cs.id << endl + << "head_branch: " << cs.head_branch << endl + << "head_sha: " << cs.head_sha << endl + << "before: " << cs.before << endl + << "after: " << cs.after << endl; + + return os; +} + +// repository +// +repository::repository (json::parser& p) +{ + p.next_expect (event::begin_object); + + // Skip unknown/uninteresting members. + // + while (p.next_expect (event::name, event::end_object)) + { + const string& n (p.name ()); + + if (n == "name") name = p.next_expect_string (); + else if (n == "full_name") full_name = p.next_expect_string (); + else if (n == "default_branch") default_branch = p.next_expect_string (); + else p.next_expect_value_skip (); + } +} + +static ostream& +operator<< (ostream& os, const repository& rep) +{ + os << "name: " << rep.name << endl + << "full_name: " << rep.full_name << endl + << "default_branch: " << rep.default_branch << endl; + + return os; +} + +// check_suite_event +// +check_suite_event::check_suite_event (json::parser& p) +{ + p.next_expect (event::begin_object); + + // Skip unknown/uninteresting members. + // + while (p.next_expect (event::name, event::end_object)) + { + const string& n (p.name ()); + + if (n == "action") action = p.next_expect_string (); + else if (n == "check_suite") check_suite = ::check_suite (p); + else if (n == "repository") repository = ::repository (p); + else p.next_expect_value_skip (); + } +} + +static ostream& +operator<< (ostream& os, const check_suite_event& cs) +{ + os << "action: " << cs.action << endl; + os << "<check_suite>" << endl << cs.check_suite; + os << "<repository>" << endl << cs.repository << endl; + + return os; +} diff --git a/mod/mod-ci-github.hxx b/mod/mod-ci-github.hxx new file mode 100644 index 0000000..a869878 --- /dev/null +++ b/mod/mod-ci-github.hxx @@ -0,0 +1,43 @@ +// file : mod/mod-ci-github.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_MOD_CI_GITHUB_HXX +#define MOD_MOD_CI_GITHUB_HXX + +#include <web/xhtml/fragment.hxx> + +#include <libbrep/types.hxx> +#include <libbrep/utility.hxx> + +#include <mod/module.hxx> +#include <mod/module-options.hxx> + +namespace brep +{ + class ci_github: public handler + { + public: + ci_github () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + ci_github (const ci_github&); + + virtual bool + handle (request&, response&); + + virtual const cli::options& + cli_options () const {return options::ci::description ();} + + private: + virtual void + init (cli::scanner&); + + private: + shared_ptr<options::ci> options_; + }; +} + +#endif // MOD_MOD_CI_GITHUB_HXX diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx index 165302d..ee2e9ce 100644 --- a/mod/mod-repository-root.cxx +++ b/mod/mod-repository-root.cxx @@ -15,6 +15,7 @@ #include <mod/module-options.hxx> #include <mod/mod-ci.hxx> +#include <mod/mod-ci-github.hxx> #include <mod/mod-submit.hxx> #include <mod/mod-upload.hxx> #include <mod/mod-builds.hxx> @@ -136,6 +137,7 @@ namespace brep ci_ (make_shared<ci> ()), #endif ci_cancel_ (make_shared<ci_cancel> ()), + ci_github_ (make_shared<ci_github> ()), upload_ (make_shared<upload> ()) { } @@ -212,6 +214,10 @@ namespace brep r.initialized_ ? r.ci_cancel_ : make_shared<ci_cancel> (*r.ci_cancel_)), + ci_github_ ( + r.initialized_ + ? r.ci_github_ + : make_shared<ci_github> (*r.ci_github_)), upload_ ( r.initialized_ ? r.upload_ @@ -244,6 +250,7 @@ namespace brep append (r, submit_->options ()); append (r, ci_->options ()); append (r, ci_cancel_->options ()); + append (r, ci_github_->options ()); append (r, upload_->options ()); return r; } @@ -292,6 +299,7 @@ namespace brep sub_init (*submit_, "submit"); sub_init (*ci_, "ci"); sub_init (*ci_cancel_, "ci-cancel"); + sub_init (*ci_github_, "ci_github"); sub_init (*upload_, "upload"); // Parse own configuration options. @@ -319,7 +327,8 @@ namespace brep "build-configs", "about", "submit", - "ci"}); + "ci", + "ci-github"}); if (find (vs.begin (), vs.end (), v) == vs.end ()) fail << what << " value '" << v << "' is invalid"; @@ -508,6 +517,13 @@ namespace brep return handle ("ci-cancel", param); } + else if (func == "ci-github") + { + if (handler_ == nullptr) + handler_.reset (new ci_github (*ci_github_)); + + return handle ("ci_github", param); + } else if (func == "upload") { if (handler_ == nullptr) diff --git a/mod/mod-repository-root.hxx b/mod/mod-repository-root.hxx index 5a57403..38f6adc 100644 --- a/mod/mod-repository-root.hxx +++ b/mod/mod-repository-root.hxx @@ -27,6 +27,7 @@ namespace brep class submit; class ci; class ci_cancel; + class ci_github; class upload; class repository_root: public handler @@ -78,6 +79,7 @@ namespace brep shared_ptr<submit> submit_; shared_ptr<ci> ci_; shared_ptr<ci_cancel> ci_cancel_; + shared_ptr<ci_github> ci_github_; shared_ptr<upload> upload_; shared_ptr<options::repository_root> options_; |