aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bdep/buildfile3
-rw-r--r--bdep/ci-parsers.cxx119
-rw-r--r--bdep/ci-parsers.hxx36
-rw-r--r--bdep/ci-types.hxx22
-rw-r--r--bdep/ci.cli64
-rw-r--r--bdep/ci.cxx42
-rw-r--r--bdep/http-service.cxx82
-rw-r--r--bdep/http-service.hxx9
-rw-r--r--tests/ci.testscript104
9 files changed, 453 insertions, 28 deletions
diff --git a/bdep/buildfile b/bdep/buildfile
index a328629..db38734 100644
--- a/bdep/buildfile
+++ b/bdep/buildfile
@@ -139,6 +139,9 @@ if $cli.configured
cli.cxx{new-options}: cli.options += \
--cxx-prologue "#include <bdep/new-parsers.hxx>"
+ cli.cxx{ci-options}: cli.options += \
+--cxx-prologue "#include <bdep/ci-parsers.hxx>"
+
# Avoid generating CLI runtime and empty inline file for help topics.
#
cli.cxx{projects-configs}: cli.options += --suppress-cli --suppress-inline
diff --git a/bdep/ci-parsers.cxx b/bdep/ci-parsers.cxx
new file mode 100644
index 0000000..2d19c4b
--- /dev/null
+++ b/bdep/ci-parsers.cxx
@@ -0,0 +1,119 @@
+// file : bdep/ci-parsers.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <sstream>
+
+#include <bdep/utility.hxx> // trim()
+
+#include <bdep/ci-parsers.hxx>
+
+#include <bdep/ci-options.hxx> // bdep::cli namespace
+
+namespace bdep
+{
+ namespace cli
+ {
+ using namespace std;
+ using namespace butl;
+
+ void parser<ci_override>::
+ parse (ci_override& r, bool& xs, scanner& s)
+ {
+ auto add = [&r] (string&& n, string&& v)
+ {
+ r.push_back (
+ manifest_name_value {move (n),
+ move (v),
+ 0, 0, 0, 0, 0, 0, 0}); // Locations, etc.
+ };
+
+ string o (s.next ());
+
+ if (!s.more ())
+ throw missing_value (o);
+
+ string v (s.next ());
+
+ if (o == "--build-email")
+ {
+ add ("build-email", move (v));
+ }
+ else if (o == "--builds")
+ {
+ add ("builds", move (v));
+ }
+ else if (o == "--override")
+ {
+ // Validate that the value has the <name>:<value> form.
+ //
+ // Note that the value semantics will be verified later, with the
+ // package_manifest::verify_overrides() called on the final list.
+ //
+ size_t p (v.find (':'));
+
+ if (p == string::npos)
+ throw invalid_value (o, v, "missing ':'");
+
+ string vn (v, 0, p);
+ string vv (v, p + 1);
+
+ // Let's trim the name and value in case they are copied from a
+ // manifest file or similar.
+ //
+ trim (vn);
+ trim (vv);
+
+ if (vn.empty ())
+ throw invalid_value (o, v, "empty value name");
+
+ add (move (vn), move (vv));
+ }
+ else if (o == "--overrides-file")
+ {
+ // Parse the manifest file into the value list.
+ //
+ // Note that the values semantics will be verified later (see above).
+ //
+ try
+ {
+ ifdstream is (v);
+
+ // In case we end up throwing invalid_value exception, its
+ // description will mention the file path as an option value.
+ // That's why we pass the empty name to the parser constructor not
+ // to mention it twice.
+ //
+ manifest_parser p (is, "" /* name */);
+
+ parse_manifest (p, r);
+ is.close ();
+ }
+ catch (const manifest_parsing& e)
+ {
+ throw invalid_value (o,
+ v,
+ string ("unable to parse: ") + e.what ());
+ }
+ catch (const io_error& e)
+ {
+ // Sanitize the exception description.
+ //
+ ostringstream os;
+ os << "unable to read: " << e;
+ throw invalid_value (o, v, os.str ());
+ }
+ }
+ else if (o == "--overrides") // Fake option.
+ {
+ throw unknown_option (o);
+ }
+ else
+ {
+ assert (false); // No other option is expected.
+ }
+
+ xs = true;
+ }
+ }
+}
diff --git a/bdep/ci-parsers.hxx b/bdep/ci-parsers.hxx
new file mode 100644
index 0000000..5a8f664
--- /dev/null
+++ b/bdep/ci-parsers.hxx
@@ -0,0 +1,36 @@
+// file : bdep/ci-parsers.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+// CLI parsers, included into the generated source files.
+//
+
+#ifndef BDEP_CI_PARSERS_HXX
+#define BDEP_CI_PARSERS_HXX
+
+#include <bdep/ci-types.hxx>
+
+namespace bdep
+{
+ namespace cli
+ {
+ class scanner;
+
+ template <typename T>
+ struct parser;
+
+ // Accumulate package manifest override values from all the override-
+ // related options (--override, --overrides-file, etc), preserving their
+ // order. Translate each option into one or more values according to the
+ // option-specific semantics.
+ //
+ template <>
+ struct parser<ci_override>
+ {
+ static void
+ parse (ci_override&, bool&, scanner&);
+ };
+ }
+}
+
+#endif // BDEP_CI_PARSERS_HXX
diff --git a/bdep/ci-types.hxx b/bdep/ci-types.hxx
new file mode 100644
index 0000000..4b2ab6d
--- /dev/null
+++ b/bdep/ci-types.hxx
@@ -0,0 +1,22 @@
+// file : bdep/ci-types.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef BDEP_CI_TYPES_HXX
+#define BDEP_CI_TYPES_HXX
+
+#include <bdep/types.hxx>
+
+#include <libbutl/manifest-types.mxx>
+
+namespace bdep
+{
+ // This type is intended for accumulating package manifest override values
+ // from all the override-related options (see ci-parsers.hxx for details).
+ //
+ class ci_override: public vector<butl::manifest_name_value>
+ {
+ };
+}
+
+#endif // BDEP_CI_TYPES_HXX
diff --git a/bdep/ci.cli b/bdep/ci.cli
index 46a3488..928e884 100644
--- a/bdep/ci.cli
+++ b/bdep/ci.cli
@@ -3,6 +3,7 @@
// license : MIT; see accompanying LICENSE file
include <bdep/project.cli>;
+include <bdep/ci-types.hxx>;
"\section=1"
"\name=bdep-ci"
@@ -55,6 +56,26 @@ namespace bdep
\cb{git(1)}, the current branch and commit id are added as the repository
URL fragment (see \l{bpkg-repository-types(1)} for details).
+ Some package manifest values can be overridden as part of the CI request
+ submission using the \cb{--override} and \cb{--overrides-file} options as
+ well as their \cb{--builds} and \cb{--build-email} shortcuts. This is
+ primarily useful for specifying alternative build configurations and/or
+ build notification emails. For example:
+
+ \
+ $ bdep ci --builds gcc
+ \
+
+ Note that manifest overrides override the entire value group that they
+ belong to. Currently, the following value groups can be overridden with
+ the \cb{build*-email} group overridden by default as if by specifying
+ an empty build email.
+
+ \
+ build-email build-{warning,error}-email
+ builds build-{include,exclude}
+ \
+
While the exact interpretation of the CI request depends on the specific
service, normally, the CI server will respond with a reference that can
be used to query the results. See \l{brep#ci Package CI} for details on
@@ -83,6 +104,49 @@ namespace bdep
"Remote repository URL for the project."
}
+ // Note: the following options are for documentation only (see --overrides
+ // below for details).
+ //
+ strings --override
+ {
+ "<name>:<value>",
+ "Package manifest value override. Repeat this option to override
+ multiple values."
+ }
+
+ path --overrides-file
+ {
+ "<file>",
+ "Read manifest value overrides from the specified manifest fragment
+ file. Repeat this option to specify multiple override files."
+ }
+
+ strings --builds
+ {
+ "<class-expr>",
+ "Shortcut for \c{\b{--override\ builds:}<class-expr>}."
+ }
+
+ string --build-email
+ {
+ "<email>",
+ "Shortcut for \c{\b{--override\ build-email:}<email>}."
+ }
+
+ // All the overrides-related options are handled with a common parser and
+ // are collected in a single manifest value list that is accessible via
+ // the --overrides option accessor. The --overrides option is "fake" in
+ // that it only serves to collect the values in a single list while
+ // preserving their order. Note that for this trick (or hack, if you
+ // will) to work, this option should come after all the options it
+ // aliases.
+ //
+ ci_override --overrides |
+ --override |
+ --overrides-file |
+ --builds |
+ --build-email;
+
string --simulate
{
"<outcome>",
diff --git a/bdep/ci.cxx b/bdep/ci.cxx
index 08193b3..8c45a11 100644
--- a/bdep/ci.cxx
+++ b/bdep/ci.cxx
@@ -4,6 +4,10 @@
#include <bdep/ci.hxx>
+#include <sstream>
+
+#include <libbutl/manifest-types.mxx>
+
#include <libbpkg/manifest.hxx>
#include <bdep/git.hxx>
@@ -144,6 +148,29 @@ namespace bdep
{
tracer trace ("ci");
+ // Create the default override.
+ //
+ vector<manifest_name_value> overrides ({
+ manifest_name_value {"build-email", "", // Name and value.
+ 0, 0, 0, 0, 0, 0, 0}}); // Locations, etc.
+
+ // Validate and append the specified overrides.
+ //
+ if (o.overrides_specified ())
+ try
+ {
+ bpkg::package_manifest::validate_overrides (o.overrides (),
+ "" /* name */);
+
+ overrides.insert (overrides.end (),
+ o.overrides ().begin (),
+ o.overrides ().end ());
+ }
+ catch (const manifest_parsing& e)
+ {
+ fail << "invalid overrides: " << e;
+ }
+
// If we are submitting the entire project, then we have two choices: we
// can list all the packages in the project or we can only do so for
// packages that were initialized in the (specified) configuration(s?).
@@ -268,6 +295,21 @@ namespace bdep
"package",
p.name.string () + '/' + p.version.string ()});
+ try
+ {
+ ostringstream os;
+ manifest_serializer s (os, "" /* name */);
+ serialize_manifest (s, overrides);
+
+ params.push_back ({parameter::file_text, "overrides", os.str ()});
+ }
+ catch (const manifest_serialization&)
+ {
+ // Values are verified by package_manifest::validate_overrides ();
+ //
+ assert (false);
+ }
+
if (o.simulate_specified ())
params.push_back ({parameter::text, "simulate", o.simulate ()});
diff --git a/bdep/http-service.cxx b/bdep/http-service.cxx
index 8b4f059..6027c9d 100644
--- a/bdep/http-service.cxx
+++ b/bdep/http-service.cxx
@@ -85,18 +85,30 @@ namespace bdep
if (!progress)
suppress_progress ();
- // Convert the submit arguments to curl's --form* options.
+ // Convert the submit arguments to curl's --form* options and cache the
+ // pointer to the file_text parameter value, if present, for writing
+ // into curl's stdin.
//
strings fos;
+ const string* file_text (nullptr);
+
for (const parameter& p: params)
{
- fos.emplace_back (p.type == parameter::file
+ if (p.type == parameter::file_text)
+ {
+ assert (file_text == nullptr);
+ file_text = &p.value;
+ }
+
+ fos.emplace_back (p.type == parameter::file ||
+ p.type == parameter::file_text
? "--form"
: "--form-string");
- fos.emplace_back (p.type == parameter::file
- ? p.name + "=@" + p.value
- : p.name + "=" + p.value);
+ fos.emplace_back (
+ p.type == parameter::file ? p.name + "=@" + p.value :
+ p.type == parameter::file_text ? p.name + "=@-" :
+ p.name + "=" + p.value);
}
// Note that it's a bad idea to issue the diagnostics while curl is
@@ -111,35 +123,43 @@ namespace bdep
//
// Start curl program.
//
- fdpipe pipe (open_pipe ()); // Text mode seems appropriate.
+ // Text mode seems appropriate.
+ //
+ fdpipe in_pipe (open_pipe ());
+ fdpipe out_pipe (file_text != nullptr ? open_pipe () : fdpipe ());
// Note that we don't specify any default timeouts, assuming that bdep
// is an interactive program and the user can always interrupt the
// command (or pass the timeout with --curl-option).
//
- process pr (start (0 /* stdin */,
- pipe /* stdout */,
- 2 /* stderr */,
- o.curl (),
- v,
- "-A", (BDEP_USER_AGENT " curl"),
+ process pr (
+ start (file_text != nullptr ? out_pipe.in.get () : 0 /* stdin */,
+ in_pipe /* stdout */,
+ 2 /* stderr */,
+ o.curl (),
+ v,
+ "-A", (BDEP_USER_AGENT " curl"),
- o.curl_option (),
+ o.curl_option (),
- // Include the response headers in the output so we
- // can get the status code/reason, content type, and
- // the redirect location.
- //
- "--include",
+ // Include the response headers in the output so we can get the
+ // status code/reason, content type, and the redirect location.
+ //
+ "--include",
- fos,
- u.string ()));
+ fos,
+ u.string ()));
// Shouldn't throw, unless something is severely damaged.
//
- pipe.out.close ();
+ in_pipe.out.close ();
+
+ if (file_text != nullptr)
+ out_pipe.in.close ();
+
+ bool io_write (false);
+ bool io_read (false);
- bool io (false);
try
{
// First we read the HTTP response status line and headers. At this
@@ -148,10 +168,22 @@ namespace bdep
// for the exception mask choice.
//
ifdstream is (
- move (pipe.in),
+ move (in_pipe.in),
fdstream_mode::skip,
ifdstream::badbit | ifdstream::failbit | ifdstream::eofbit);
+ if (file_text != nullptr)
+ {
+ ofdstream os (move (out_pipe.out));
+ os << *file_text;
+ os.close ();
+
+ // Indicate to the potential IO error handling that we are done with
+ // writing.
+ //
+ file_text = nullptr;
+ }
+
// Parse and return the HTTP status code. Return 0 if the argument is
// invalid.
//
@@ -428,7 +460,7 @@ namespace bdep
// Presumably the child process failed and issued diagnostics so let
// finish() try to deal with that first.
//
- io = true;
+ (file_text != nullptr ? io_write : io_read) = true;
}
// Handle all parsing errors, including the manifest_parsing exception
// that inherits from the runtime_error exception.
@@ -455,7 +487,7 @@ namespace bdep
dr << info << "reference: " << *reference;
}
- finish (o.curl (), pr, io);
+ finish (o.curl (), pr, io_read, io_write);
assert (!message.empty ());
diff --git a/bdep/http-service.hxx b/bdep/http-service.hxx
index 75fd603..b4eb4f2 100644
--- a/bdep/http-service.hxx
+++ b/bdep/http-service.hxx
@@ -5,7 +5,7 @@
#ifndef BDEP_HTTP_SERVICE_HXX
#define BDEP_HTTP_SERVICE_HXX
-#include <libbutl/manifest-parser.mxx>
+#include <libbutl/manifest-types.mxx>
#include <bdep/types.hxx>
#include <bdep/utility.hxx>
@@ -16,11 +16,12 @@ namespace bdep
{
namespace http_service
{
- // If type is file, then the value is a path to be uploaded.
+ // If type is file, then the value is a path to be uploaded. If type is
+ // file_text, then the value is a file content to be uploaded.
//
struct parameter
{
- enum {text, file} type;
+ enum {text, file, file_text} type;
string name;
string value;
};
@@ -40,6 +41,8 @@ namespace bdep
// POST method. Use the multipart/form-data content type if any files are
// uploaded and application/x-www-form-urlencoded otherwise.
//
+ // Note: currently only one file_text parameter can be specified.
+ //
// On success, return the response manifest message and reference (if
// present, see below) and the rest of the manifest values, if any. Issue
// diagnostics and fail if anything goes wrong or the response manifest
diff --git a/tests/ci.testscript b/tests/ci.testscript
index 7bd6172..986648f 100644
--- a/tests/ci.testscript
+++ b/tests/ci.testscript
@@ -138,6 +138,110 @@ windows = ($cxx.target.class == 'windows')
error: remote git repository URL '$repository#frag' already has fragment
EOE
}
+
+ : overrides
+ :
+ {
+ +$clone_root_prj
+ +$init -C @cfg &prj-cfg/***
+
+ : valid
+ :
+ : Here we only test that bdep-ci does not fail for valid overrides. It
+ : seems to be impossible to verify the submitted overrides manifest.
+ :
+ {
+ $clone_prj;
+
+ cat <<EOI >=overrides.manifest;
+ : 1
+ builds: all
+ EOI
+
+ $* --overrides-file overrides.manifest \
+ --build-email 'foo@example.com' \
+ --override 'build-error-email: error@example.com' \
+ --override 'builds: &gcc' \
+ --override 'build-include: linux*' \
+ --override 'build-exclude: *' 2>>~"%EOE%"
+ submitting to $server
+ %.*
+ %.*CI request is queued.*%
+ %reference: .+%
+ EOE
+ }
+
+ : invalid-option
+ :
+ {
+ +$clone_prj
+
+ : overrides-file
+ :
+ {
+ +$clone_prj
+
+ : reading
+ :
+ {
+ $clone_prj;
+
+ $* --overrides-file overrides.manifest 2>>~%EOE% != 0
+ %error: invalid value 'overrides.manifest' for option '--overrides-file': unable to read: .+%
+ EOE
+ }
+
+ : parsing
+ :
+ {
+ $clone_prj;
+
+ cat <<EOI >=overrides.manifest;
+ builds: all
+ EOI
+
+ $* --overrides-file overrides.manifest 2>>EOE != 0
+ error: invalid value 'overrides.manifest' for option '--overrides-file': unable to parse: 1:1: error: format version pair expected
+ EOE
+ }
+ }
+
+ : override
+ :
+ {
+ $clone_prj;
+
+ $* --override 'all' 2>>EOE != 0
+ error: invalid value 'all' for option '--override': missing ':'
+ EOE
+ }
+
+ : override-pair
+ :
+ : Each override option is valid by its own, but the resulting overrides
+ : manifest is invalid.
+ :
+ {
+ $clone_prj;
+
+ $* --override 'builds: all' --override 'builds: default : &gcc' 2>>EOE != 0
+ error: invalid overrides: invalid package builds in 'default : &gcc': unexpected underlying class set
+ EOE
+ }
+
+ : overrides
+ :
+ : This is a fake option (see ci.cli for details).
+ :
+ {
+ $clone_prj;
+
+ $* --overrides 'all' 2>>EOE != 0
+ error: unknown option '--overrides'
+ EOE
+ }
+ }
+ }
}
: multi-pkg