fileapi: Add protocol v1 support for client-specific query files

Add support for client-owned stateless query files.  These allow clients
to *own* requests for major object versions and get all those recognized
by CMake.

Issue: #18398

Help/manual/cmake-file-api.7.rst

@@ -31,7 +31,8 @@ It has the following subdirectories:
 
 ``query/``
   Holds query files written by clients.
-  These may be `v1 Shared Stateless Query Files`_.
+  These may be `v1 Shared Stateless Query Files`_ or
+  `v1 Client Stateless Query Files`_.
 
 ``reply/``
   Holds reply files written by CMake whenever it runs to generate a build
@@ -62,6 +63,27 @@ Files of this form are stateless shared queries not owned by any specific
 client.  Once created they should not be removed without external client
 coordination or human intervention.
 
+v1 Client Stateless Query Files
+-------------------------------
+
+Client stateless query files allow clients to create owned requests for
+major versions of the `Object Kinds`_ and get all requested versions
+recognized by the CMake that runs.
+
+Clients may create owned requests by creating empty files in
+client-specific query subdirectories.  The form is::
+
+  <build>/.cmake/api/v1/query/client-<client>/<kind>-v<major>
+
+where ``client-`` is literal, ``<client>`` is a string uniquely
+identifying the client, ``<kind>`` is one of the `Object Kinds`_,
+``-v`` is literal, and ``<major>`` is the major version number.
+Each client must choose a unique ``<client>`` identifier via its
+own means.
+
+Files of this form are stateless queries owned by the client ``<client>``.
+The owning client may remove them at any time.
+
 v1 Reply Index File
 -------------------
 
@@ -107,7 +129,14 @@ The reply index file contains a JSON object:
                            "version": { "major": 1, "minor": 0 },
                            "jsonFile": "<file>" },
       "<unknown>": { "error": "unknown query file" },
-      "...": {}
+      "...": {},
+      "client-<client>": {
+        "<kind>-v<major>": { "kind": "<kind>",
+                             "version": { "major": 1, "minor": 0 },
+                             "jsonFile": "<file>" },
+        "<unknown>": { "error": "unknown query file" },
+        "...": {}
+      }
     }
   }
 
@@ -162,6 +191,26 @@ The members are:
     containing a string with an error message indicating that the
     query file is unknown.
 
+  ``client-<client>``
+    A member of this form appears for each client-owned directory
+    holding `v1 Client Stateless Query Files`_.
+    The value is a JSON object mirroring the content of the
+    ``query/client-<client>/`` directory.  The members are of the form:
+
+    ``<kind>-v<major>``
+      A member of this form appears for each of the
+      `v1 Client Stateless Query Files`_ that CMake recognized as a
+      request for object kind ``<kind>`` with major version ``<major>``.
+      The value is a `v1 Reply File Reference`_ to the corresponding
+      reply file for that object kind and version.
+
+    ``<unknown>``
+      A member of this form appears for each of the
+      `v1 Client Stateless Query Files`_ that CMake did not recognize.
+      The value is a JSON object with a single ``error`` member
+      containing a string with an error message indicating that the
+      query file is unknown.
+
 After reading the reply index file, clients may read the other
 `v1 Reply Files`_ it references.
 

Source/cmFileAPI.cxx

@@ -2,6 +2,7 @@
    file Copyright.txt or https://cmake.org/licensing for details.  */
 #include "cmFileAPI.h"
 
+#include "cmAlgorithms.h"
 #include "cmCryptoHash.h"
 #include "cmSystemTools.h"
 #include "cmTimestamp.h"
@@ -42,7 +43,9 @@ void cmFileAPI::ReadQueries()
 
   // Read the queries and save for later.
   for (std::string& query : queries) {
-    if (!cmFileAPI::ReadQuery(query, this->TopQuery.Known)) {
+    if (cmHasLiteralPrefix(query, "client-")) {
+      this->ReadClient(query);
+    } else if (!cmFileAPI::ReadQuery(query, this->TopQuery.Known)) {
       this->TopQuery.Unknown.push_back(std::move(query));
     }
   }
@@ -176,6 +179,21 @@ bool cmFileAPI::ReadQuery(std::string const& query,
   return false;
 }
 
+void cmFileAPI::ReadClient(std::string const& client)
+{
+  // Load queries for the client.
+  std::string clientDir = this->APIv1 + "/query/" + client;
+  std::vector<std::string> queries = this->LoadDir(clientDir);
+
+  // Read the queries and save for later.
+  Query& clientQuery = this->ClientQueries[client];
+  for (std::string& query : queries) {
+    if (!this->ReadQuery(query, clientQuery.Known)) {
+      clientQuery.Unknown.push_back(std::move(query));
+    }
+  }
+}
+
 Json::Value cmFileAPI::BuildReplyIndex()
 {
   Json::Value index(Json::objectValue);
@@ -184,7 +202,12 @@ Json::Value cmFileAPI::BuildReplyIndex()
   index["cmake"] = this->BuildCMake();
 
   // Reply to all queries that we loaded.
-  index["reply"] = this->BuildReply(this->TopQuery);
+  Json::Value& reply = index["reply"] = this->BuildReply(this->TopQuery);
+  for (auto const& client : this->ClientQueries) {
+    std::string const& clientName = client.first;
+    Query const& clientQuery = client.second;
+    reply[clientName] = this->BuildReply(clientQuery);
+  }
 
   // Move our index of generated objects into its field.
   Json::Value& objects = index["objects"] = Json::arrayValue;

Source/cmFileAPI.h

@@ -77,6 +77,9 @@ private:
   /** The content of the top-level query directory.  */
   Query TopQuery;
 
+  /** The content of each "client-$client" query directory.  */
+  std::map<std::string, Query> ClientQueries;
+
   /** Reply index object generated for object kind/version.
       This populates the "objects" field of the reply index.  */
   std::map<Object, Json::Value> ReplyIndexObjects;
@@ -91,6 +94,7 @@ private:
 
   static bool ReadQuery(std::string const& query,
                         std::vector<Object>& objects);
+  void ReadClient(std::string const& client);
 
   Json::Value BuildReplyIndex();
   Json::Value BuildCMake();

Tests/RunCMake/FileAPI/ClientStateless-check.cmake

@@ -0,0 +1,15 @@
+set(expect
+  query
+  query/client-foo
+  query/client-foo/__test-v1
+  query/client-foo/__test-v2
+  query/client-foo/__test-v3
+  query/client-foo/unknown
+  reply
+  reply/__test-v1-[0-9a-f]+.json
+  reply/__test-v2-[0-9a-f]+.json
+  reply/index-[0-9.T-]+.json
+  )
+check_api("^${expect}$")
+
+check_python(ClientStateless)

Tests/RunCMake/FileAPI/ClientStateless-check.py

@@ -0,0 +1,26 @@
+from check_index import *
+
+def check_reply(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == ["client-foo"]
+    check_reply_client_foo(r["client-foo"])
+
+def check_reply_client_foo(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == ["__test-v1", "__test-v2", "__test-v3", "unknown"]
+    check_index__test(r["__test-v1"], 1, 3)
+    check_index__test(r["__test-v2"], 2, 0)
+    check_error(r["__test-v3"], "unknown query file")
+    check_error(r["unknown"], "unknown query file")
+
+def check_objects(o):
+    assert is_list(o)
+    assert len(o) == 2
+    check_index__test(o[0], 1, 3)
+    check_index__test(o[1], 2, 0)
+
+assert is_dict(index)
+assert sorted(index.keys()) == ["cmake", "objects", "reply"]
+check_cmake(index["cmake"])
+check_reply(index["reply"])
+check_objects(index["objects"])

Tests/RunCMake/FileAPI/ClientStateless-prep.cmake

@@ -0,0 +1,5 @@
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/__test-v1" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/__test-v2" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/__test-v3" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/unknown" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/reply/object-to-be-deleted.json" "")

Tests/RunCMake/FileAPI/DuplicateStateless-check.cmake

@@ -0,0 +1,20 @@
+set(expect
+  query
+  query/__test-v1
+  query/__test-v2
+  query/__test-v3
+  query/client-foo
+  query/client-foo/__test-v1
+  query/client-foo/__test-v2
+  query/client-foo/__test-v3
+  query/client-foo/unknown
+  query/query.json
+  query/unknown
+  reply
+  reply/__test-v1-[0-9a-f]+.json
+  reply/__test-v2-[0-9a-f]+.json
+  reply/index-[0-9.T-]+.json
+  )
+check_api("^${expect}$")
+
+check_python(DuplicateStateless)

Tests/RunCMake/FileAPI/DuplicateStateless-check.py

@@ -0,0 +1,31 @@
+from check_index import *
+
+def check_reply(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == ["__test-v1", "__test-v2", "__test-v3", "client-foo", "query.json", "unknown"]
+    check_index__test(r["__test-v1"], 1, 3)
+    check_index__test(r["__test-v2"], 2, 0)
+    check_error(r["__test-v3"], "unknown query file")
+    check_reply_client_foo(r["client-foo"])
+    check_error(r["query.json"], "unknown query file")
+    check_error(r["unknown"], "unknown query file")
+
+def check_reply_client_foo(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == ["__test-v1", "__test-v2", "__test-v3", "unknown"]
+    check_index__test(r["__test-v1"], 1, 3)
+    check_index__test(r["__test-v2"], 2, 0)
+    check_error(r["__test-v3"], "unknown query file")
+    check_error(r["unknown"], "unknown query file")
+
+def check_objects(o):
+    assert is_list(o)
+    assert len(o) == 2
+    check_index__test(o[0], 1, 3)
+    check_index__test(o[1], 2, 0)
+
+assert is_dict(index)
+assert sorted(index.keys()) == ["cmake", "objects", "reply"]
+check_cmake(index["cmake"])
+check_reply(index["reply"])
+check_objects(index["objects"])

Tests/RunCMake/FileAPI/DuplicateStateless-prep.cmake

@@ -0,0 +1,10 @@
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/__test-v1" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/__test-v2" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/__test-v3" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/query.json" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/unknown" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/__test-v1" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/__test-v2" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/__test-v3" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/unknown" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/reply/object-to-be-deleted.json" "")

Tests/RunCMake/FileAPI/EmptyClient-check.cmake

@@ -0,0 +1,9 @@
+set(expect
+  query
+  query/client-foo
+  reply
+  reply/index-[0-9.T-]+.json
+  )
+check_api("^${expect}$")
+
+check_python(EmptyClient)

Tests/RunCMake/FileAPI/EmptyClient-check.py

@@ -0,0 +1,20 @@
+from check_index import *
+
+def check_reply(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == ["client-foo"]
+    check_reply_client_foo(r["client-foo"])
+
+def check_reply_client_foo(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == []
+
+def check_objects(o):
+    assert is_list(o)
+    assert len(o) == 0
+
+assert is_dict(index)
+assert sorted(index.keys()) == ["cmake", "objects", "reply"]
+check_cmake(index["cmake"])
+check_reply(index["reply"])
+check_objects(index["objects"])

Tests/RunCMake/FileAPI/EmptyClient-prep.cmake

@@ -0,0 +1,2 @@
+file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/reply/object-to-be-deleted.json" "")

Tests/RunCMake/FileAPI/MixedStateless-check.cmake

@@ -0,0 +1,16 @@
+set(expect
+  query
+  query/__test-v1
+  query/__test-v3
+  query/client-foo
+  query/client-foo/__test-v2
+  query/client-foo/unknown
+  query/query.json
+  reply
+  reply/__test-v1-[0-9a-f]+.json
+  reply/__test-v2-[0-9a-f]+.json
+  reply/index-[0-9.T-]+.json
+  )
+check_api("^${expect}$")
+
+check_python(MixedStateless)

Tests/RunCMake/FileAPI/MixedStateless-check.py

@@ -0,0 +1,27 @@
+from check_index import *
+
+def check_reply(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == ["__test-v1", "__test-v3", "client-foo", "query.json"]
+    check_index__test(r["__test-v1"], 1, 3)
+    check_error(r["__test-v3"], "unknown query file")
+    check_reply_client_foo(r["client-foo"])
+    check_error(r["query.json"], "unknown query file")
+
+def check_reply_client_foo(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == ["__test-v2", "unknown"]
+    check_index__test(r["__test-v2"], 2, 0)
+    check_error(r["unknown"], "unknown query file")
+
+def check_objects(o):
+    assert is_list(o)
+    assert len(o) == 2
+    check_index__test(o[0], 1, 3)
+    check_index__test(o[1], 2, 0)
+
+assert is_dict(index)
+assert sorted(index.keys()) == ["cmake", "objects", "reply"]
+check_cmake(index["cmake"])
+check_reply(index["reply"])
+check_objects(index["objects"])

Tests/RunCMake/FileAPI/MixedStateless-prep.cmake

@@ -0,0 +1,6 @@
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/__test-v1" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/__test-v2" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/__test-v3" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/query.json" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/client-foo/unknown" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/reply/object-to-be-deleted.json" "")

Tests/RunCMake/FileAPI/RunCMakeTest.cmake

@@ -36,5 +36,9 @@ endfunction()
 
 run_cmake(Nothing)
 run_cmake(Empty)
+run_cmake(EmptyClient)
 run_cmake(Stale)
 run_cmake(SharedStateless)
+run_cmake(ClientStateless)
+run_cmake(MixedStateless)
+run_cmake(DuplicateStateless)