From b1de3f25d4d1e993fbeabe8d9fd2107d2d895a07 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 22 Jul 2024 16:22:49 +0000 Subject: [PATCH] Update after first round of testing --- codelabs/how-to-add-a-client-action.md | 42 +++++++++++------- .../how-to-add-a-client-action/codelab.json | 4 +- .../how-to-add-a-client-action/index.html | 44 ++++++++++++------- codelabs/how-to-add-a-flow.md | 17 ++++--- codelabs/how-to-add-a-flow/codelab.json | 4 +- codelabs/how-to-add-a-flow/index.html | 17 ++++--- compose.yaml | 20 +++++++-- 7 files changed, 99 insertions(+), 49 deletions(-) diff --git a/codelabs/how-to-add-a-client-action.md b/codelabs/how-to-add-a-client-action.md index a995f7d9f..9979d4ebb 100644 --- a/codelabs/how-to-add-a-client-action.md +++ b/codelabs/how-to-add-a-client-action.md @@ -22,16 +22,17 @@ The code you'll be touching is mostly Python. You don't need to be an expert to ## Defining the input and outputs for your Client Action -Duration: 2 +Duration: 5 -The input and outputs of your Client Action are its public interface. The input is provided by the Flow when calling your action. The outputs are what you will provide back so it can continue to process. +The input and output of your Client Action are its public interface. The input is provided by the Flow when calling your action. The output is what will be provided back so it can continue to process. You'll need to define a ```.proto``` and an equivalent ```RDFValue``` for them. Let's go through [an example](https://github.com/google/grr/blob/master/grr/proto/grr_response_proto/dummy.proto): +[https://github.com/google/grr/blob/master/grr/proto/grr_response_proto/dummy.proto](https://github.com/google/grr/blob/master/grr/proto/grr_response_proto/dummy.proto) ```protobuf message DummyRequest { optional string action_input = 1; @@ -45,9 +46,10 @@ message DummyResult { Next, let's add the corresponding ```RDFValue``` [classes](https://github.com/google/grr/blob/master/grr/core/grr_response_core/lib/rdfvalues/dummy.py). They inherit from ```RDFProtoStruct```, and must have the ```protobuf``` property set. If your proto depends on other ```RDFValues``` (e.g. other protos), you should add them to the list of dependencies in ```rdf_deps``` ([example](https://github.com/google/grr/blob/a6f1b31abfe82794b7d82fa8d54d8bd94bfed1bb/grr/server/grr_response_server/gui/api_plugins/flow.py#L466C3-L466C11)). +[https://github.com/google/grr/blob/master/grr/core/grr_response_core/lib/rdfvalues/dummy.py](https://github.com/google/grr/blob/master/grr/core/grr_response_core/lib/rdfvalues/dummy.py) ```python #!/usr/bin/env python """The various Dummy example rdfvalues.""" @@ -70,11 +72,11 @@ class DummyResult(rdf_structs.RDFProtoStruct): rdf_deps = [] ``` -An important detail is that for Client Actions, we prefer adding the class to a separate file (see below). This is because we need to import it in both the server and the client. For Flows, we usually prefer to define them close to the Flow class definition themselves (see more on [Adding Flows](../how-to-add-a-flow/index.html)). +An important detail is that for Client Actions, we prefer adding the class to a separate file. This is because we need to import it in both the server and the client. For Flows, we usually prefer to define them close to the Flow class definition themselves (see more on [Adding Flows](../how-to-add-a-flow/index.html)). ## Writing the Client Action class -Duration: 3 +Duration: 5 Client Actions are classes that inherit from [```ActionPlugin```](https://github.com/google/grr/blob/a6f1b31abfe82794b7d82fa8d54d8bd94bfed1bb/grr/client/grr_response_client/actions.py#L78). The class must override: @@ -92,6 +94,7 @@ Whith that in mind, let's write our Dummy client action. The action will be simp Our action will be the same for Linux and MacOS (Unix). So, we'll add it to the [common/shared directory](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/dummy.py). +[https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/dummy.py](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/dummy.py) ```python class Dummy(actions.ActionPlugin): """Returns the received string.""" @@ -113,13 +116,14 @@ class Dummy(actions.ActionPlugin): ``` ### Writing the platform-specific (Windows) implementation -For Windows, we will write a [dedicated action](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/windows/dummy.py). It'll live in the windows folder. For +For Windows, we will write a [dedicated action](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/windows/dummy.py). It'll live in the windows folder. +[https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/windows/dummy.py](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/windows/dummy.py) ```python class Dummy(actions.ActionPlugin): """Returns the received string.""" @@ -142,7 +146,7 @@ class Dummy(actions.ActionPlugin): ## Writing the Client Action unit tests -Duration: 1 +Duration: 4 ### Unix implementation unit tests @@ -154,8 +158,9 @@ IMPORTANT: ALWAYS submit your tests together with the code. Whenever a Client Action reaches the end of the processing, the Client sends a Status message back to the server. This is usually the last message sent to the server. -On our unit tests below, we're including this message in the tests. On the successful test case, we also test the status code is ```OK```. On the failure scenario, we test that it returned a ```GENERIC_ERROR``` (since we raise a ```RuntimeException``` in the action). ```Status``` messages can have other values as well (e.g. reached a limit). You can take a look at other actions and the proto to see how else it can be used. +In our unit tests below, we're including this message in the tests. In the successful test case, we also test the status code is ```OK```. In the failure scenario, we test that it returned a ```GENERIC_ERROR``` (since we raise a ```RuntimeException``` in the action). ```Status``` messages can have other values as well (e.g. reached a limit). You can take a look at other actions and the proto to see how else it can be used. +[https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/dummy_test.py](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/dummy_test.py) ```python def testDummyReceived(self): action_request = rdf_dummy.DummyRequest(action_input="banana") @@ -193,6 +198,7 @@ On our unit tests below, we're including this message in the tests. On the succe For Windows, we're doing exactly the [same as the above](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/windows/dummy_test.py), except for some extra assertions that match the Client Action behavior on Windows. +[https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/windows/dummy_test.py](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/windows/dummy_test.py) ```python def testDummyReceived(self): action_request = rdf_dummy.DummyRequest(action_input="banana") @@ -229,15 +235,18 @@ def testDummyReceived(self): ## Registering your Client Action -Duration: 1 +Duration: 5 -Now that we have an implemented and tested action, we can register it so it is available to be called (cl/559042698)! Hooray! +Now that we have an implemented and tested action, we can register it so it is available to be called! Hooray! For that you need to add it to the following places: -To the [```server_stubs```](https://github.com/google/grr/blob/master/grr/server/grr_response_server/server_stubs.py#L54). The stub must have the same name, ```in_``` and ```out_rdfvalues``` as your class declared on the previous step. +To the [```server_stubs```](https://github.com/google/grr/blob/master/grr/server/grr_response_server/server_stubs.py). The stub must have the same name, ```in_``` and ```out_rdfvalues``` as your class declared on the previous step. +[https://github.com/google/grr/blob/master/grr/server/grr_response_server/server_stubs.py](https://github.com/google/grr/blob/master/grr/server/grr_response_server/server_stubs.py) ```python +from grr_response_core.lib.rdfvalues import dummy as rdf_dummy + class Dummy(ClientActionStub): """Dummy example. Reads a message and sends it back.""" @@ -245,12 +254,14 @@ class Dummy(ClientActionStub): out_rdfvalues = [rdf_dummy.DummyResult] ``` -To the [```action_registry```](https://github.com/google/grr/blob/a6f1b31abfe82794b7d82fa8d54d8bd94bfed1bb/grr/server/grr_response_server/action_registry.py#L10) and [```registry_init```](https://github.com/google/grr/blob/a6f1b31abfe82794b7d82fa8d54d8bd94bfed1bb/grr/client/grr_response_client/client_actions/registry_init.py#L74). The registry key should also have the same class name. Note that, we're registering this implementation for both Linux and Darwin, but will do something different for Windows (see next step). +To the [```action_registry```](https://github.com/google/grr/blob/a6f1b31abfe82794b7d82fa8d54d8bd94bfed1bb/grr/server/grr_response_server/action_registry.py#L10) and [```registry_init```](https://github.com/google/grr/blob/a6f1b31abfe82794b7d82fa8d54d8bd94bfed1bb/grr/client/grr_response_client/client_actions/registry_init.py#L74). The registry key should also have the same class name. Note that, we're registering this implementation for both Linux and Darwin, but will do something different for Windows (see 'Registering platform-specific' section below). +[https://github.com/google/grr/blob/master/grr/server/grr_response_server/action_registry.py](https://github.com/google/grr/blob/master/grr/server/grr_response_server/action_registry.py) ```python "Dummy": server_stubs.Dummy, ``` +[https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/registry_init.py](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/registry_init.py) ```python client_actions.Register("Dummy", dummy.Dummy) ``` @@ -259,12 +270,13 @@ client_actions.Register("Dummy", dummy.Dummy) Notice that when registering it, we'll register it on the [```Windows```] [section](https://github.com/google/grr/blob/a6f1b31abfe82794b7d82fa8d54d8bd94bfed1bb/grr/client/grr_response_client/client_actions/registry_init.py#L87) of the code. +[https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/registry_init.py](https://github.com/google/grr/blob/master/grr/client/grr_response_client/client_actions/registry_init.py) ```python client_actions.Register("Dummy", win_dummy.Dummy) ``` ## ... And now to call it from a Flow! -Duration: 1 +Duration: 3 That's it, your Client Action is complete! Now you can start using it in an existing or new Flow! See [Adding a Flow](../how-to-add-a-flow/index.html) diff --git a/codelabs/how-to-add-a-client-action/codelab.json b/codelabs/how-to-add-a-client-action/codelab.json index 6479c712c..30a20d31a 100644 --- a/codelabs/how-to-add-a-client-action/codelab.json +++ b/codelabs/how-to-add-a-client-action/codelab.json @@ -3,9 +3,9 @@ "format": "html", "prefix": "https://storage.googleapis.com", "mainga": "UA-49880327-14", - "updated": "2024-07-22T12:24:26Z", + "updated": "2024-07-22T16:19:36Z", "id": "how-to-add-a-client-action", - "duration": 9, + "duration": 23, "title": "How to Add a Client Action", "authors": "Tati, Dan", "summary": "How to Add a Client Action", diff --git a/codelabs/how-to-add-a-client-action/index.html b/codelabs/how-to-add-a-client-action/index.html index 2ce6cf180..61272e176 100644 --- a/codelabs/how-to-add-a-client-action/index.html +++ b/codelabs/how-to-add-a-client-action/index.html @@ -40,11 +40,12 @@ - -

The input and outputs of your Client Action are its public interface. The input is provided by the Flow when calling your action. The outputs are what you will provide back so it can continue to process.

- +[https://github.com/google/grr/blob/master/grr/server/grr_response_server/flows/general/dummy_test.py](https://github.com/google/grr/blob/master/grr/server/grr_response_server/flows/general/dummy_test.py) ```python # Mocks the Dummy Client Action. class DummyActionReturnsOnce(actions.ActionPlugin): @@ -265,7 +269,7 @@ if __name__ == "__main__": ## Registering your Flow -Duration: 1 +Duration: 4 Now that we have an implemented and tested action, we can register it so it is available to be called! Hooray! @@ -273,12 +277,14 @@ For that you need to add it to the following places. To [```registry_init```](https://github.com/google/grr/blob/a6f1b31abfe82794b7d82fa8d54d8bd94bfed1bb/grr/server/grr_response_server/flows/general/registry_init.py#L11) to be used regularly. +[https://github.com/google/grr/blob/master/grr/server/grr_response_server/flows/general/registry_init.py](https://github.com/google/grr/blob/master/grr/server/grr_response_server/flows/general/registry_init.py) ```python from grr_response_server.flows.general import dummy ``` To [```utils```](https://github.com/google/grr/blob/a6f1b31abfe82794b7d82fa8d54d8bd94bfed1bb/api_client/python/grr_api_client/utils.py#L27) to be used by the api client. +[https://github.com/google/grr/blob/master/api_client/python/grr_api_client/utils.py](https://github.com/google/grr/blob/master/api_client/python/grr_api_client/utils.py) ```python from grr_response_proto import dummy_pb2 ``` @@ -291,12 +297,13 @@ When a new Flow is registered, you will also need to add the new protos to the U ## Writing the Flow end to end tests -Duration: 1 +Duration: 5 GRR also has end to end tests. In your test functions, you can use ```RunFlowAndWait``` to run your Flow with the arguments you want, and then consult/test Flow results and other properties such as whether logs were written or not. Here's an [example end to end test](https://github.com/google/grr/blob/master/grr/test/grr_response_test/end_to_end_tests/tests/dummy.py) for our Dummy Flow. +[https://github.com/google/grr/blob/master/grr/test/grr_response_test/end_to_end_tests/tests/dummy.py](https://github.com/google/grr/blob/master/grr/test/grr_response_test/end_to_end_tests/tests/dummy.py) ```python #!/usr/bin/env python """End to end tests for GRR dummy example Flow.""" @@ -360,7 +367,7 @@ class TestDummyWindows(test_base.EndToEndTest): ## ... And now to call it from a Flow! -Duration: 1 +Duration: 3 That's it, your Flow is complete! Now you can trigger it locally. diff --git a/codelabs/how-to-add-a-flow/codelab.json b/codelabs/how-to-add-a-flow/codelab.json index 2c5b0adc3..1cd9bfcad 100644 --- a/codelabs/how-to-add-a-flow/codelab.json +++ b/codelabs/how-to-add-a-flow/codelab.json @@ -3,9 +3,9 @@ "format": "html", "prefix": "https://storage.googleapis.com", "mainga": "UA-49880327-14", - "updated": "2024-07-22T12:33:18Z", + "updated": "2024-07-22T16:21:24Z", "id": "how-to-add-a-flow", - "duration": 13, + "duration": 27, "title": "How to Add a Flow", "authors": "Tati, Dan", "summary": "How to Add a Flow", diff --git a/codelabs/how-to-add-a-flow/index.html b/codelabs/how-to-add-a-flow/index.html index 85969d06b..4b3c15251 100644 --- a/codelabs/how-to-add-a-flow/index.html +++ b/codelabs/how-to-add-a-flow/index.html @@ -47,6 +47,7 @@

Writing the FlowArgs and FlowResult .proto

First, write the .proto messages for your new Flow.

+

https://github.com/google/grr/blob/master/grr/proto/grr_response_proto/dummy.proto

message DummyArgs {
   optional string flow_input = 1;
 }
@@ -58,6 +59,7 @@ 
Write the FlowArgs and FlowResult RDFValue classes<
 
Next, let's add the corresponding RDFValue classes. They inherit from RDFProtoStruct, and must have the protobuf property set. If your proto depends on other RDFValues (e.g. other protos), you should add them to the list of dependencies in rdf_deps (example).

 
+
https://github.com/google/grr/blob/master/grr/core/grr_response_core/lib/rdfvalues/dummy.py

 
class DummyArgs(rdf_structs.RDFProtoStruct):
   """Request for Dummy action."""
 
@@ -75,7 +77,7 @@ 
Write the FlowArgs and FlowResult RDFValue classes<
 
       
     
-      
+      
         
Flows are classes that inherit from FlowBase. The class must override:

 
    
 
  • args_type and result_types properties: these are the public interface for your Flow - the input provided from the user to your Flow, and what the Flow will output back to them. An important detail here is that this value must be an RDFValue.
    • 
@@ -96,6 +98,7 @@ 
    Write the FlowArgs and FlowResult RDFValue classes<
 
  • End: This is the last state executed by the Flow before it completes. If you need something done at the end, consider adding it here.
    • 
 

 
Whith all of that in mind, let's write our Dummy Flow. When it starts, it will simply read the input string, modify it, and send it to the Client Action. When the Client Action finishes, we'll also append to the string and return our Flow results.

+
https://github.com/google/grr/blob/master/grr/server/grr_response_server/flows/general/dummy.py

 
class Dummy(flow_base.FlowBase):
   """A mechanism to send a string to the client and back.
 
@@ -156,13 +159,14 @@ 
Write the FlowArgs and FlowResult RDFValue classes<
 
       
     
-      
+      
         
Here's an example test for our very simple Flow. We are aiming at covering the different scenarios that can happen, where things can go wrong, and what we expect to happen in these cases.

 
In the first test, we're testing the happy path. Here, the Flow has an input string, provides it to the mocked client, the client returns as we expect and we test the Flow result.

 
On the second test, we're testing the Start method argument validation. Here, it shouldn't matter which client action or mock you provide, as the client action will never be called.

 
The third test uses a client mock to test the next_state function, and that it fails under the right conditions.

 
+
https://github.com/google/grr/blob/master/grr/server/grr_response_server/flows/general/dummy_test.py

 
# Mocks the Dummy Client Action.
 class DummyActionReturnsOnce(actions.ActionPlugin):
   """Sends a single Reply (like real action would)."""
@@ -257,13 +261,15 @@ 
Write the FlowArgs and FlowResult RDFValue classes<
 
       
     
-      
+      
         
Now that we have an implemented and tested action, we can register it so it is available to be called! Hooray!

 
For that you need to add it to the following places.

 
To registry_init to be used regularly.

+
https://github.com/google/grr/blob/master/grr/server/grr_response_server/flows/general/registry_init.py

 
from grr_response_server.flows.general import dummy
 

 
To utils to be used by the api client.

+
https://github.com/google/grr/blob/master/api_client/python/grr_api_client/utils.py

 
from grr_response_proto import dummy_pb2
 

 
When a new Flow is registered, you will also need to add the new protos to the UI code. You can do that by running:

@@ -272,9 +278,10 @@ 
Write the FlowArgs and FlowResult RDFValue classes<
 
       
     
-      
+      
         
GRR also has end to end tests.

 
In your test functions, you can use RunFlowAndWait to run your Flow with the arguments you want, and then consult/test Flow results and other properties such as whether logs were written or not. Here's an example end to end test for our Dummy Flow.

+
https://github.com/google/grr/blob/master/grr/test/grr_response_test/end_to_end_tests/tests/dummy.py

 
#!/usr/bin/env python
 """End to end tests for GRR dummy example Flow."""
 
@@ -338,7 +345,7 @@ 
Write the FlowArgs and FlowResult RDFValue classes<
 
       
     
-      
+      
         
That's it, your Flow is complete! Now you can trigger it locally.

 
    
 
  1. Make sure GRR and Fleetspeak Servers and Clients are running! See more on the development setup page.
    2. 
diff --git a/compose.yaml b/compose.yaml
index 2911711ab..72f489647 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -27,6 +27,7 @@ services:
       retries: 10
 
   grr-admin-ui:
+    build: .
     image: ghcr.io/google/grr:latest
     container_name: grr-admin-ui
     hostname: admin-ui
@@ -57,6 +58,13 @@ services:
       test: "/configs/healthchecks/grr-admin-ui.sh"
       timeout: 10s
       retries: 10
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./grr
+          target: /usr/src/grr/grr
+          ignore:
+            - client/
 
   grr-fleetspeak-frontend:
     image: ghcr.io/google/grr:latest
@@ -150,8 +158,10 @@ services:
     develop:
       watch:
         - action: sync+restart
-          path: ./grr/server
-          target: /usr/src/grr/grr/server
+          path: ./grr
+          target: /usr/src/grr/grr
+          ignore:
+            - client/
 
   grr-client:
     build: .
@@ -187,8 +197,10 @@ services:
     develop:
       watch:
         - action: sync+restart
-          path: ./grr/client
-          target: /usr/src/grr/grr/client
+          path: ./grr
+          target: /usr/src/grr/grr
+          ignore:
+            - server/
 
 volumes:
   db_data: