diff options
| author | Karen Arutyunov <karen@codesynthesis.com> | 2016-10-05 16:44:37 +0300 |
|---|---|---|
| committer | Karen Arutyunov <karen@codesynthesis.com> | 2016-10-17 21:07:56 +0300 |
| commit | 639e679564f8401a6aa5d9233cda0c1618c40fd3 (patch) | |
| tree | cf2e09a590081f9bc53982cc3edf7a5c405d365e | |
| parent | 842df807e704d07410fbff166e957478154ef3e6 (diff) | |
Add msvc-filter utility
| -rw-r--r-- | .gitignore | 5 | ||||
| -rw-r--r-- | INSTALL | 54 | ||||
| -rw-r--r-- | LICENSE | 20 | ||||
| -rw-r--r-- | NEWS | 0 | ||||
| -rw-r--r-- | README | 0 | ||||
| -rw-r--r-- | TODO | 7 | ||||
| -rw-r--r-- | build/.gitignore | 1 | ||||
| -rw-r--r-- | build/bootstrap.build | 19 | ||||
| -rw-r--r-- | build/root.build | 14 | ||||
| -rw-r--r-- | buildfile | 28 | ||||
| -rw-r--r-- | manifest | 14 | ||||
| -rwxr-xr-x | msvc-common | 99 | ||||
| -rw-r--r-- | msvc-filter.cxx | 416 | ||||
| -rw-r--r-- | version | 1 |
14 files changed, 536 insertions, 142 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b2ffb54 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# Compiler/linker output. +# +msvc-filter +*.d +*.o @@ -4,6 +4,11 @@ Setup Setting Scripts --------------- +@@ Replace this section with a proper package build/install instructions when + the package name is finalized. +@@ Interestingly INSTALL files for our packages provide instuctions how to + build the package but not how to install it. + The "driver" scripts that you will be invoking are cl-NN, link-NN, lib-NN, etc., where NN is the Visual Studio version (e.g., 11, 12, 14, etc). There are also the "configuration" scripts, msvc-NN, which provide the Visual @@ -11,10 +16,13 @@ Studio/SDK location and configuration (in a way similar to vcvars32.bat). There is also a bunch of other helper scripts that you will not need to modify or invoke directly. -All of these scripts should reside in the same directory. In particular, you -cannot copy, say, cl-NN to /usr/local/bin/ while leaving the rest in some -other directory -- this will not work. What you can do, however, is create -symlinks to the driver scripts in /usr/local/bin/ -- this will work. +It is also required to build msvc-filter utility. To do that run build2 in the +project's directory. + +All of these scripts and utility should reside in the same directory. In +particular, you cannot copy, say, cl-NN to /usr/local/bin/ while leaving the +rest in some other directory -- this will not work. What you can do, however, +is create symlinks to the driver scripts in /usr/local/bin/ -- this will work. If you only need to make the scripts usable by a single user, then the easiest approach is to add the script's directory to your PATH in, say, .bashrc, for @@ -212,41 +220,3 @@ built with more recent Visual Studio versions) you may want to add the following line to your .bashrc or similar: export WINEDEBUG=fixme-all - -Certain executions of the cl compiler (e.g., /EP /showIncludes) are very -slow unless wineserver is started and a special voodoo dance is performed: - -# Kill existing Wine processes if any. -# -$ pkill wineserver -$ pkill .exe - -# Verify no Wine processes are running. -# -$ pgrep wineserver -$ pgrep .exe - -# Prepare test file. -# -cat > test.cpp -#include <iostream> -^D - -# Get time without wineserver. -# -$ time cl-NN /EP /showIncludes test.cpp >/dev/null - -# Start wineserver. -# -$ wineserver -p -$ pgrep wineserver - -# The voodoo part: the first execution after starting wineserver will -# hang, but all subsequent ones should be much faster. -# -$ cl-NN /EP /showIncludes test.cpp >/dev/null -^C - -# Get time with wineserver. -# -$ time cl-NN /EP /showIncludes test.cpp >/dev/null @@ -0,0 +1,20 @@ +Copyright (c) 2014-2016 Code Synthesis Ltd + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @@ -6,15 +6,10 @@ @@ Had to rename Windows.h to windows.h; /showIncludes seem to retain naming from #include directive. Also WinBase.h, WinUser.h, WinNls.h, ... nsed? -@@ Requires bash 4, GNU sed 4.2.2 or later (-z), realpath dirpath +@@ Requires bash 4, realpath dirpath @@ Still need wineserver for /EP, etc. otherwise long pause. @@ What about 64-bit? cl-64 or some such? @@ There is link POSIX command. link.exe? cl-14.exe? - -@@ The /showIncludes run is very slow most likely due to the sed-based path - processing (but verify). Perhaps we should rewrite it in C++ with - translated path caching, etc. Test result: sed/winepath results in ~x5 - slowdown. diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000..225c27f --- /dev/null +++ b/build/.gitignore @@ -0,0 +1 @@ +config.build diff --git a/build/bootstrap.build b/build/bootstrap.build new file mode 100644 index 0000000..0d2bcd5 --- /dev/null +++ b/build/bootstrap.build @@ -0,0 +1,19 @@ +# file : build/bootstrap.build +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +project = msvc-linux + +using build@0.4.0 + +version = 0.1.0-a1 +revision = 0 + +dist.package = $project-$version + +if ($revision != 0) + dist.package += +$revision + +using config +using dist +using install diff --git a/build/root.build b/build/root.build new file mode 100644 index 0000000..62bf08a --- /dev/null +++ b/build/root.build @@ -0,0 +1,14 @@ +# file : build/root.build +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +cxx.std = 14 + +using cxx + +hxx{*}: extension = +ixx{*}: extension = ixx +txx{*}: extension = txx +cxx{*}: extension = cxx + +cxx.poptions =+ -I$src_root diff --git a/buildfile b/buildfile new file mode 100644 index 0000000..6d1467c --- /dev/null +++ b/buildfile @@ -0,0 +1,28 @@ +# file : buildfile +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +define sh: file +sh{*}: extension = +sh{*}: install = bin/ + +s = cl-11 cl-12 cl-14 cl-14u2 \ + lib-11 lib-12 lib-14 lib-14u2 \ + link-11 link-12 link-14 link-14u2 \ + msvc-11 msvc-12 msvc-14 msvc-14u2 \ + mt-11 mt-12 mt-14 mt-14u2 \ + rc-11 rc-12 rc-14 rc-14u2 \ + \ + msvc-cl-common msvc-common msvc-lib-common \ + msvc-link-common msvc-mt-common msvc-rc-common + +./: exe{msvc-filter} sh{$s} doc{INSTALL LICENSE NEWS README version} \ + file{manifest} + +import libs = libbutl%lib{butl} + +exe{msvc-filter}: cxx{msvc-filter} $libs + +# Don't install INSTALL file. +# +doc{INSTALL}@./: install = false diff --git a/manifest b/manifest new file mode 100644 index 0000000..f1da8ca --- /dev/null +++ b/manifest @@ -0,0 +1,14 @@ +: 1 +name: msvc-linux # @@ Shouldn't we change it to some more generic name? +version: 0.1.0-a1 +summary: MS Visual C++ driver scripts +license: MIT +tags: msvc, vc, c++, script +description-file: README +changes-file: NEWS +url: https://build2.org # @@ Use the proper url and email. +email: users@build2.org +requires: c++14 +depends: * build2 >= 0.4.0 +depends: * bpkg >= 0.4.0 +depends: libbutl == 0.5.0-a1 diff --git a/msvc-common b/msvc-common index 096b17f..479ec3f 100755 --- a/msvc-common +++ b/msvc-common @@ -33,107 +33,18 @@ function split_translate () # <length> <option-path> # The <diag> argument should be 1 or 2. It indicates whether the diagnostics # is sent to stdout (1) or stderr (2). # -# Note that if <exe> returns non-zero exit status, then this function calls -# exit, not return. It also clears the ERR trap and overrides the EXIT trap. -# All this pretty much means it should be the last statement in a call. -# function msvc_exec () # <diag> <exe> <arg>... { - local diag=$1 - shift - - local exe="$1" + local diag="$1" shift - # Assemble the arguments in an array to store in case they contain spaces. - # - local args=() - - while [ $# -gt 0 ]; do - args=("${args[@]}" "$1") - shift - done - - # Translate absolute Windows paths back to POSIX. The hard part here is to - # determing the end of the path. For example, the error location has the - # 'X:\...\foo(10):' form. However, we cannot assume that '(' ends the path; - # remember 'Program Files (x86)'. - # - # To sidestep this whole mess we are going to use this trick: instead of - # translating the whole path we will only translate its directory part, that - # is the longest part that still ends with the directory separator. We will - # also still recognize ':' and ''' as path terminators as well as space if - # it is the first character in the component. - # - # We also pass the path through realpath in order to get the actual path - # rather than Wine's dosdevices links. - # - - # First delimit paths that we need to translate with NUL characters. - # - local s1="s#[A-Za-z]:[\\/]([^ ':][^':]*[\\/])*#\x00&\x00#g" - - # Next translate the paths (note the -z sed option). The last xargs call - # does two things: it removes the newline added by realpath and adds the - # trailing slash removed by realpath. - # - # Substitution useful for debugging: #/bin/echo -n '&'# - # - local s2="s#^[A-Za-z]:[\\/]([^ ':][^':]*[\\/])*#winepath -u0 '&' | \ -xargs -0 realpath -z | xargs -0 -I{} /bin/echo -n {}/#e" - - # Finally, get rid of the NUL characters. While at it, also kill Windows - # CR (0x0d). - # - local s3="s#\x00##g;s#\x0d##g" - - # For testing/debugging: - # - #cat input | sed -re "$s1" | sed -z -re "$s2" | sed -re "$s3" - # Suppress Wine noise. # export WINEDEBUG=fixme-all - # Create a temporary named pipe. - # - local pipe - pipe="$(mktemp -u)" - mkfifo $pipe - trap "{ rm $pipe; }" EXIT - - if [ $diag -eq 1 ]; then - wine "$exe" "${args[@]}" 2>&1 1>$pipe & - sed -re "$s1" $pipe | sed -z -re "$s2" | sed -re "$s3" - else - # For some reason Wine is really slow when we redirect stdout to - # /dev/null. A lot slower than redirecting it to a file. This is observed - # with Wine 1.7, 1.8, and 1.9. As an admittedly bizarre workaround we are - # going to channel the output via a fifo. Yes, it does help, a lot. - # - local opipe - opipe="$(mktemp -u)" - mkfifo $opipe - trap "{ rm $pipe $opipe; }" EXIT - - cat $opipe & - local pid=$! - - wine "$exe" "${args[@]}" 2>$pipe >$opipe & - sed -re "$s1" $pipe | sed -z -re "$s2" | sed -re "$s3" 1>&2 - - # Wait for cat. If it fails then the ERR trap will terminate us. - # - wait $pid - fi - - # Wait for the wine process and exit with its exit status if it's not - # zero. Don't you just hate bash sometimes? I sure do. + # Filter diagnostics output replacing absolute Windows paths with their + # POSIX mapping. If <diag> is 1 then both stdout and stderr output are read + # and filtered. # - trap - ERR - wait $! - local r=$? - if [ $r -ne 0 ]; then - exit $r - fi + "$(dirname $(realpath ${BASH_SOURCE[0]}))/msvc-filter" "$diag" wine "$@" } diff --git a/msvc-filter.cxx b/msvc-filter.cxx new file mode 100644 index 0000000..0a06202 --- /dev/null +++ b/msvc-filter.cxx @@ -0,0 +1,416 @@ +// file : msvc-filter/msvc-filter.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <sys/time.h> // timeval +#include <sys/select.h> + +#include <ios> // ios_base::failure +#include <string> +#include <cstring> // strchr() +#include <ostream> +#include <iostream> +#include <algorithm> // max() +#include <system_error> +#include <unordered_map> + +#include <butl/path> // path::traits::realize() +#include <butl/utility> // alpha() +#include <butl/process> +#include <butl/fdstream> + +using namespace std; +using namespace butl; + +// Cached mapping of Windows paths to the corresponding POSIX paths. +// +static unordered_map<string, string> path_cache; + +inline static bool +path_separator (char c) +{ + return c == '\\' || c == '/'; +} + +// Replace absolute Windows paths encountered in the line with their POSIX +// representation. Write the result followed by a newline to the stream. Throw +// ostream::failure on IO failures, system_error on others. +// +static void +filter (const char* s, size_t n, ostream& os) +{ + // Strip the terminating 0xOD character if present. + // + if (n > 0 && s[n - 1] == '\r') + --n; + + // Translate absolute Windows paths back to POSIX. The hard part here is to + // determing the end of the path. For example, the error location has the + // 'X:\...\foo(10):' form. However, we cannot assume that '(' ends the path; + // remember 'Program Files (x86)'. + // + // To sidestep this whole mess we are going to use this trick: instead of + // translating the whole path we will only translate its directory part, + // that is, the longest part that still ends with the directory + // separator. We will also still recognize ':' and '\'' as path terminators + // as well as space if it is the first character in the component. + // + // We also pass the path through realpath in order to get the actual path + // rather than Wine's dosdevices links. + // + const char* b (s); + const char* e (s + n); + + for (;;) + { + const char* p (b); + + // Line tail should be at least 3 characters long to contain an absolute + // Windows path. + // + bool no_path (e - b < 3); + + if (!no_path) + { + // An absolute path must begin with [A-Za-z]:[\/] (like C:\). + // + const char* pe (e - 3); + + for (; p != pe; ++p) + { + if (alpha (p[0]) && p[1] == ':' && path_separator (p[2])) + break; + } + + no_path = p == pe; + } + + // Bail out if we reached the end of the line with no path found. + // + if (no_path) + { + os.write (b, e - b); + os.put ('\n'); + break; + } + + os.write (b, p - b); // Print characters that preceed the path. + + b = p; // Beginnig of the path. + const char* pe (p + 3); // End of the last path component. + + for (p = pe; p != e; ++p) + { + char c (*p); + if (c == ':' || c == '\'' || (p == pe && c == ' ')) + break; + + if (path_separator (c)) + pe = p + 1; + } + + // Convert the Windows directory path to POSIX. First check if the mapping + // is already cached. + // + string d (b, pe - b); + auto i (path_cache.find (d)); + + b = pe; + + if (i == path_cache.end ()) + { + const char* args[] = {"winepath", "-u", d.c_str (), nullptr}; + + // Redirect stderr to /dev/null not to mess the output (read more in + // main()). + // + process pr (args, 0, -1, -2); + ifdstream is (pr.in_ofd); + + string pd; + getline (is, pd); + is.close (); + + // It's unknown what can cause winepath to fail. At least not a + // non-existent path. Anyway will consider it fatal. + // + if (!pr.wait ()) + throw system_error (ECHILD, system_category ()); + + try + { + path::traits::realize (pd); + + assert (!pd.empty ()); + + // Restore the trailing slash. + // + if (pd.back () != '/') + pd += '/'; + + } + catch (const invalid_path&) + { + // The path doesn't exist. Let's keep it as provided by winepath. + } + + i = path_cache.emplace (move (d), move (pd)).first; + } + + os.write (i->second.c_str (), i->second.size ()); + } +} + +int +main (int argc, char* argv[]) +try +{ + auto print_usage = [argv]() + { + cerr << "usage: " << argv[0] + << " <diag> <wine-path> <program-path> [arguments]" << endl; + }; + + if (argc < 2) + { + cerr << "error: diag stream file descriptor expected" << endl; + print_usage (); + return 1; + } + + string d (argv[1]); + if (d != "1" && d != "2") + { + cerr << "error: invalid diag stream file descriptor" << endl; + print_usage (); + return 1; + } + + size_t diag (stoi (d)); + + if (argc < 3) + { + cerr << "error: wine path expected" << endl; + print_usage (); + return 1; + } + + if (argc < 4) + { + cerr << "error: program path expected" << endl; + print_usage (); + return 1; + } + + // After arguments are validated we will not be printing error messages on + // failures not to mess the filtered output of the child Windows process. + // Note that in the case of a failure the text from STDERR can be treated by + // the calling process as a build tool diagnostics so our message most + // likelly would be misinterpreted. + // + // The reason we still print error message on the arguments parsing failure + // is that it is likely to be a program misuse rather than runtime error. + // + + // If diag is 1 then both stdout and stderr of the child process are read and + // filtered (achieved by redirecting stdout to stderr). Otherwise the data + // read from child stdout is proxied to own stdout (sounds redundant but + // prevents Windows process from writing to /dev/null directly which is known + // to be super slow). The filtered data is written to the diag file + // descriptor. + // + process pr (const_cast <const char**> (&argv[2]), 0, diag == 1 ? 2 : -1, -1); + + // Stream to filter from. + // + ifdstream isf (pr.in_efd, fdstream_mode::non_blocking); + + // Stream to proxy from. + // + ifdstream isp (diag == 1 ? -1 : pr.in_ofd, fdstream_mode::non_blocking); + + ostream& osf (diag == 1 ? cout : cerr); // Stream to filter to. + ostream* osp (diag == 1 ? nullptr : &cout); // Stream to proxy to. + + // The presense of proxy input and output streams must be correlated. + // + assert (isp.is_open () == (osp != nullptr)); + + // Will be using ostream::write() solely, so badbit is the only one which + // needs to be set. + // + osf.exceptions (ostream::badbit); + + if (osp != nullptr) + osp->exceptions (ostream::badbit); + + const size_t nbuf (8192); + char buf[nbuf + 1]; // Reserve one extra for terminating '\0'. + + bool terminated (false); // Required for Wine bug workaround (see below). + string line; // Incomplete line. + + while (isf.is_open () || isp.is_open ()) + { + // Use timeout to workaround the wineserver bug: sometimes the file + // descriptor that corresponds to the redirected STDERR of a Windows + // process is not closed when that process is terminated. So if STDERR is + // redirected to a pipe (as in our case) the reading peer does not receive + // EOF and hangs forever. We will consider the no-data 100 ms period for an + // exited process to represent such a situation. + // + // Note that it is wineserver who owns the corresponding file descriptor + // not a wine process which runs the Windows program. + // + // Note also that some implementations of select() can modify the timeout + // value so it is essential to reinitialize it prior to every select() + // call. + // + timeval timeout {0, 100000}; + + fd_set rd; + FD_ZERO (&rd); + + if (isf.is_open ()) + FD_SET (isf.fd (), &rd); + + if (isp.is_open ()) + FD_SET (isp.fd (), &rd); + + int r (select (max (isf.fd (), isp.fd ()) + 1, + &rd, + nullptr, + nullptr, + &timeout)); + + if (r == -1) + { + if (errno == EINTR) + continue; + + throw system_error (errno, system_category ()); + } + + // Timeout occured. Apply wineserver bug workaround if required. + // + bool status; + if (r == 0 && pr.try_wait (status)) + { + if (!status) + // Handle the child failure outside the loop. + // + break; + + // Presumably end of the data reached. + // + if (!terminated) + { + // We don't know when the process exited. It possibly wasn't writing + // to the output for quite a long time before terminating a nanosecond + // ago. But let's wait for another timeout to be sure that the process + // has terminated a long (enough) time ago. + // + terminated = true; + continue; + } + + break; + } + + // Proxy the data if requested. + // + if (FD_ISSET (isp.fd (), &rd)) + { + for (;;) + { + // The only leagal way to read from non-blocking ifdstream. + // + streamsize n (isp.readsome (buf, nbuf)); + + if (isp.eof ()) + { + // End of the data to be proxied reached. + // + isp.close (); + break; + } + + if (n == 0) + break; // No data available, try later. + + assert (osp != nullptr); + osp->write (buf, n); + } + } + + // Read & filter. + // + if (FD_ISSET (isf.fd (), &rd)) + { + for (;;) + { + // The only leagal way to read from non-blocking ifdstream. + // + streamsize n (isf.readsome (buf, nbuf)); + + if (isf.eof ()) + { + // End of the data to be filtered reached. + // + isf.close (); + break; + } + + if (n == 0) + break; // No data available, try later. + + // Filter buffer content line by line. Concatenate the line with an + // incomplete one if produced on the previous iteration. Save the last + // line if incomplete (not terminated with '\n'). + // + buf[n] = '\0'; + const char* b (buf); + + for (;;) + { + const char* le (strchr (b, '\n')); + + if (le == nullptr) + { + line += b; + break; + } + + if (!line.empty ()) + { + line.append (b, le - b); + filter (line.c_str (), line.size (), osf); + line.clear (); + } + else + filter (b, le - b, osf); + + b = le + 1; // Skip the newline character. + } + } + } + } + + if (!line.empty ()) + filter (line.c_str (), line.size (), osf); + + isf.close (); + isp.close (); + + return pr.wait () ? 0 : pr.status; +} +catch (const ios_base::failure&) +{ + return 1; +} +// Also handles process_error exception (derived from system_error). +// +catch (const system_error&) +{ + return 1; +} @@ -0,0 +1 @@ +0.1.0-a1 |