Handle see-through groups with dynamic members in dist

2 files changed, 126 insertions, 25 deletions

diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx
index da96215..cd88eac 100644
--- a/libbuild2/dist/operation.cxx
+++ b/libbuild2/dist/operation.cxx
@@ -463,41 +463,89 @@ namespace build2
         // Note that we are not showing progress here (e.g., "N targets to
         // distribute") since it will be useless (too fast).
         //
-        for (const auto& pt: ctx.targets)
+        auto see_through = [] (const target& t)
         {
-          file* ft (pt->is_a<file> ());
-
-          if (ft == nullptr) // Not a file.
-            continue;
+          return ((t.type ().flags & target_type::flag::see_through) ==
+                  target_type::flag::see_through);
+        };
 
-          if (ft->dir.sub (src_root))
+        auto collect = [&trace, &dist_var,
+                        &src_root, &out_root] (const file& ft)
+        {
+          if (ft.dir.sub (src_root))
           {
             // Include unless explicitly excluded.
             //
-            if (const path* v = cast_null<path> ((*ft)[dist_var]))
+            if (const path* v = cast_null<path> (ft[dist_var]))
             {
               if (v->string () == "false")
               {
-                l5 ([&]{trace << "excluding " << *ft;});
-                continue;
+                l5 ([&]{trace << "excluding " << ft;});
+                return false;
               }
             }
 
-            files.push_back (ft);
+            return true;
           }
-          else if (ft->dir.sub (out_root))
+          else if (ft.dir.sub (out_root))
           {
             // Exclude unless explicitly included.
             //
-            if (const path* v = cast_null<path> ((*ft)[dist_var]))
+            if (const path* v = cast_null<path> (ft[dist_var]))
+            {
+              if (v->string () != "false")
+              {
+                l5 ([&]{trace << "including " << ft;});
+                return true;
+              }
+            }
+
+            return false;
+          }
+          else
+            return false; // Out of project.
+        };
+
+        for (const auto& pt: ctx.targets)
+        {
+          // Collect see-through groups if they are marked with dist=true.
+          //
+          // Note that while it's possible that only their certain members are
+          // marked as such (e.g., via a pattern), we will still require
+          // dist=true on the group itself (and potentially dist=false on some
+          // of its members) for such cases because we don't want to update
+          // every see-through group only to discover that most of them don't
+          // have anything to distribute.
+          //
+          if (see_through (*pt))
+          {
+            if (const path* v = cast_null<path> ((*pt)[dist_var]))
             {
               if (v->string () != "false")
               {
-                l5 ([&]{trace << "including " << *ft;});
-                files.push_back (ft);
+                l5 ([&]{trace << "including group " << *pt;});
+                files.push_back (pt.get ());
               }
             }
+
+            continue;
           }
+
+          file* ft (pt->is_a<file> ());
+
+          if (ft == nullptr) // Not a file.
+            continue;
+
+          // Skip member of see-through groups since after dist_* their list
+          // can be incomplete (or even bogus, e.g., the "representative
+          // sample"). Instead, we will collect them during perfrom_update
+          // below.
+          //
+          if (ft->group != nullptr && see_through (*ft->group))
+            continue;
+
+          if (collect (*ft))
+            files.push_back (ft);
         }
 
         // Make sure what we need to distribute is up to date.
@@ -537,6 +585,50 @@ namespace build2
                               1    /* diag (failures only) */,
                               prog /* progress */);
 
+          // Replace see-through groups (which now should have their members
+          // resolved) with members.
+          //
+          for (auto i (files.begin ()); i != files.end (); )
+          {
+            const target& t (i->as<target> ());
+            if (see_through (t))
+            {
+              group_view gv (t.group_members (a)); // Go directly.
+
+              if (gv.members == nullptr)
+                fail << "unable to resolve see-through group " << t
+                     << " members";
+
+              i = files.erase (i); // Drop the group itself.
+
+              for (size_t j (0); j != gv.count; ++j)
+              {
+                if (const target* m = gv.members[j])
+                {
+                  if (const file* ft = m->is_a<file> ())
+                  {
+                    // Note that a rule may only link-up its members to groups
+                    // if/when matched (for example, the cli.cxx{} group). It
+                    // feels harmless for us to do the linking here.
+                    //
+                    if (ft->group == nullptr)
+                      const_cast<file*> (ft)->group = &t;
+                    else
+                      assert (ft->group == &t); // Sanity check.
+
+                    if (collect (*ft))
+                    {
+                      i = files.insert (i, ft); // Insert instead of the group.
+                      i++;                      // Stay after the group.
+                    }
+                  }
+                }
+              }
+            }
+            else
+              ++i;
+          }
+
           if (op_update.operation_post != nullptr)
             op_update.operation_post (ctx, {}, true /* inner */);
 
@@ -585,7 +677,7 @@ namespace build2
 
       for (size_t i (0), n (files.size ()); i != n; ++i)
       {
-        const file& t (*files[i].as<target> ().is_a<file> ());
+        const file& t (files[i].as<target> ().as<file> ()); // Only files.
 
         // Figure out where this file is inside the target directory.
         //
diff --git a/libbuild2/dist/rule.cxx b/libbuild2/dist/rule.cxx
index 736490e..320d17a 100644
--- a/libbuild2/dist/rule.cxx
+++ b/libbuild2/dist/rule.cxx
@@ -30,10 +30,21 @@ namespace build2
       const dir_path& src_root (rs.src_path ());
       const dir_path& out_root (rs.out_path ());
 
-      // If we can, go inside see-through groups.
+      // Note that we don't go inside see-through groups since the members for
+      // dist_* may be incomplete (or even bogus, e.g., the "representative
+      // sample"). Instead, for see-through groups our plan is as follows:
       //
-      for (prerequisite_member pm:
-             group_prerequisite_members (a, t, members_mode::maybe))
+      // 1. Here we match them as groups (so that we still match all their
+      //    prerequisites).
+      //
+      // 2. In dist_project() we collect them along with files after dist_*
+      //    but before perform_update. Here we also skip files that are
+      //    members of see-through groups (which we may still get).
+      //
+      // 3. During perform_update we collect all the see-through group
+      //    members, similar to files on step (2).
+      //
+      for (const prerequisite& p: group_prerequisites (t))
       {
         // Note: no exclusion tests, we want all of them (and see also the
         // dist_include() override). But if we don't ignore post hoc ones
@@ -41,12 +52,12 @@ namespace build2
         // by the post-pass).
         //
         lookup l; // Ignore any operation-specific values.
-        if (include (a, t, pm, &l) == include_type::posthoc)
+        if (include (a, t, p, &l) == include_type::posthoc)
           continue;
 
         // Skip prerequisites imported from other projects.
         //
-        if (pm.proj ())
+        if (p.proj)
           continue;
 
         // We used to always search and match but that resulted in the
@@ -65,14 +76,12 @@ namespace build2
         // @@ Note that this is still an issue in a custom dist rule.
         //
         const target* pt (nullptr);
-        if (pm.is_a<file> ())
+        if (p.is_a<file> ())
         {
-          pt = pm.load ();
+          pt = p.target.load ();
 
           if (pt == nullptr)
           {
-            const prerequisite& p (pm.prerequisite);
-
             // Search for an existing target or existing file in src.
             //
             // Note: see also similar code in match_postponed() below.
@@ -106,7 +115,7 @@ namespace build2
           }
         }
         else
-          pt = &pm.search (t);
+          pt = &search (t, p);
 
         // Don't match targets that are outside of our project.
         //