[Playground] "always_run" flag in examples' tag (#25928)

* Add 'always_run' flag for examples * Add 'always_run' flag to protocol * [Fixup] Add 'always_run' flag for examples * Return 'AlwaysRun' flag in gRPC responses * Frontend support for 'alwaysRun' flag, preserve pipeline options on page refresh * Fix a test * Update docstring --------- Co-authored-by: Alexey Inkin <alexey.inkin@akvelon.com>
apache · Mar 28, 2023 · 469101d · 469101d
1 parent 567bdaa
commit 469101d
Show file tree

Hide file tree

Showing 19 changed files with 744 additions and 674 deletions.
diff --git a/learning/tour-of-beam/backend/playground_api/api/v1/api.pb.go b/learning/tour-of-beam/backend/playground_api/api/v1/api.pb.go
diff --git a/playground/api/v1/api.proto b/playground/api/v1/api.proto
@@ -194,6 +194,7 @@ message PrecompiledObject{
   // Link to the example in the Beam repository
   string url_vcs = 14;
   string url_notebook = 15;
+  bool always_run = 16;
 }
 
 // Categories represent the array of messages with sdk and categories at this sdk

diff --git a/playground/backend/internal/api/v1/api.pb.go b/playground/backend/internal/api/v1/api.pb.go
diff --git a/playground/backend/internal/db/dto/precompiled_object.go b/playground/backend/internal/db/dto/precompiled_object.go
@@ -30,6 +30,7 @@ type ObjectInfo struct {
 	Link            string
 	UrlVCS          string
 	UrlNotebook     string
+	AlwaysRun       bool
 	Multifile       bool
 	ContextLine     int32
 	DefaultExample  bool

diff --git a/playground/backend/internal/db/entity/example.go b/playground/backend/internal/db/entity/example.go
@@ -29,6 +29,7 @@ type ExampleEntity struct {
 	SchVer      *datastore.Key `datastore:"schVer"`
 	UrlVCS      string         `datastore:"urlVCS"`
 	UrlNotebook string         `datastore:"urlNotebook"`
+	AlwaysRun   bool           `datastore:"alwaysRun"`
 }
 
 type PrecompiledObjectEntity struct {

diff --git a/playground/backend/internal/db/mapper/precompiled_object_mapper.go b/playground/backend/internal/db/mapper/precompiled_object_mapper.go
@@ -44,6 +44,7 @@ func (pom *PrecompiledObjectMapper) ToObjectInfo(exampleDTO *dto.ExampleDTO) *dt
 		Link:            exampleDTO.Example.Path,
 		UrlVCS:          exampleDTO.Example.UrlVCS,
 		UrlNotebook:     exampleDTO.Example.UrlNotebook,
+		AlwaysRun:       exampleDTO.Example.AlwaysRun,
 		Multifile:       exampleDTO.HasMultiFiles(),
 		ContextLine:     exampleDTO.GetContextLine(),
 		DefaultExample:  exampleDTO.IsDefault(),
@@ -80,9 +81,13 @@ func (pom *PrecompiledObjectMapper) ToArrayCategories(catalogDTO *dto.CatalogDTO
 	}
 	sdkCategories := make([]*pb.Categories, 0)
 	for sdkName, categories := range sdkToCategories {
-		sdkCategory := pb.Categories{Sdk: pb.Sdk(pb.Sdk_value[sdkName]), Categories: make([]*pb.Categories_Category, 0)}
+		sdkCategory := pb.Categories{
+			Sdk:        pb.Sdk(pb.Sdk_value[sdkName]),
+			Categories: make([]*pb.Categories_Category, 0),
+		}
 		for categoryName, precompiledObjects := range categories {
-			putPrecompiledObjectsToCategory(categoryName, &precompiledObjects, &sdkCategory)
+			category := precompiledObjectsToCategory(categoryName, precompiledObjects)
+			sdkCategory.Categories = append(sdkCategory.Categories, category)
 		}
 		sdkCategories = append(sdkCategories, &sdkCategory)
 	}
@@ -123,6 +128,7 @@ func (pom *PrecompiledObjectMapper) ToPrecompiledObj(exampleDTO *dto.ExampleDTO)
 		Link:            exampleDTO.Example.Path,
 		UrlVcs:          exampleDTO.Example.UrlVCS,
 		UrlNotebook:     exampleDTO.Example.UrlNotebook,
+		AlwaysRun:       exampleDTO.Example.AlwaysRun,
 		Multifile:       exampleDTO.HasMultiFiles(),
 		ContextLine:     exampleDTO.GetContextLine(),
 		DefaultExample:  exampleDTO.IsDefault(),
@@ -196,13 +202,13 @@ func appendPrecompiledObject(objectInfo dto.ObjectInfo, sdkToCategories *dto.Sdk
 	categoryToPrecompiledObjects[categoryName] = append(objects, objectInfo)
 }
 
-// putPrecompiledObjectsToCategory adds categories with precompiled objects to protobuf object
-func putPrecompiledObjectsToCategory(categoryName string, precompiledObjects *dto.PrecompiledObjects, sdkCategory *pb.Categories) {
+// precompiledObjectsToCategory create category protobuf object from precompiled objects
+func precompiledObjectsToCategory(categoryName string, precompiledObjects dto.PrecompiledObjects) *pb.Categories_Category {
 	category := pb.Categories_Category{
 		CategoryName:       categoryName,
 		PrecompiledObjects: make([]*pb.PrecompiledObject, 0),
 	}
-	for _, object := range *precompiledObjects {
+	for _, object := range precompiledObjects {
 		category.PrecompiledObjects = append(category.PrecompiledObjects, &pb.PrecompiledObject{
 			CloudPath:       object.CloudPath,
 			Name:            object.Name,
@@ -212,6 +218,7 @@ func putPrecompiledObjectsToCategory(categoryName string, precompiledObjects *dt
 			Link:            object.Link,
 			UrlVcs:          object.UrlVCS,
 			UrlNotebook:     object.UrlNotebook,
+			AlwaysRun:       object.AlwaysRun,
 			Multifile:       object.Multifile,
 			ContextLine:     object.ContextLine,
 			DefaultExample:  object.DefaultExample,
@@ -221,7 +228,7 @@ func putPrecompiledObjectsToCategory(categoryName string, precompiledObjects *dt
 			Datasets:        object.Datasets,
 		})
 	}
-	sdkCategory.Categories = append(sdkCategory.Categories, &category)
+	return &category
 }
 
 // getCloudPath returns the cloud path by example entity

diff --git a/playground/frontend/playground_components/lib/src/api/v1/api.pb.dart b/playground/frontend/playground_components/lib/src/api/v1/api.pb.dart
@@ -1066,6 +1066,7 @@ class PrecompiledObject extends $pb.GeneratedMessage {
     ..pc<Dataset>(13, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'datasets', $pb.PbFieldType.PM, subBuilder: Dataset.create)
     ..aOS(14, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'urlVcs')
     ..aOS(15, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'urlNotebook')
+    ..aOB(16, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'alwaysRun')
     ..hasRequiredFields = false
   ;
 
@@ -1086,6 +1087,7 @@ class PrecompiledObject extends $pb.GeneratedMessage {
     $core.Iterable<Dataset>? datasets,
     $core.String? urlVcs,
     $core.String? urlNotebook,
+    $core.bool? alwaysRun,
   }) {
     final _result = create();
     if (cloudPath != null) {
@@ -1133,6 +1135,9 @@ class PrecompiledObject extends $pb.GeneratedMessage {
     if (urlNotebook != null) {
       _result.urlNotebook = urlNotebook;
     }
+    if (alwaysRun != null) {
+      _result.alwaysRun = alwaysRun;
+    }
     return _result;
   }
   factory PrecompiledObject.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
@@ -1278,6 +1283,15 @@ class PrecompiledObject extends $pb.GeneratedMessage {
   $core.bool hasUrlNotebook() => $_has(14);
   @$pb.TagNumber(15)
   void clearUrlNotebook() => clearField(15);
+
+  @$pb.TagNumber(16)
+  $core.bool get alwaysRun => $_getBF(15);
+  @$pb.TagNumber(16)
+  set alwaysRun($core.bool v) { $_setBool(15, v); }
+  @$pb.TagNumber(16)
+  $core.bool hasAlwaysRun() => $_has(15);
+  @$pb.TagNumber(16)
+  void clearAlwaysRun() => clearField(16);
 }
 
 class Categories_Category extends $pb.GeneratedMessage {

diff --git a/playground/frontend/playground_components/lib/src/api/v1/api.pbjson.dart b/playground/frontend/playground_components/lib/src/api/v1/api.pbjson.dart
@@ -324,11 +324,12 @@ const PrecompiledObject$json = const {
     const {'1': 'datasets', '3': 13, '4': 3, '5': 11, '6': '.api.v1.Dataset', '10': 'datasets'},
     const {'1': 'url_vcs', '3': 14, '4': 1, '5': 9, '10': 'urlVcs'},
     const {'1': 'url_notebook', '3': 15, '4': 1, '5': 9, '10': 'urlNotebook'},
+    const {'1': 'always_run', '3': 16, '4': 1, '5': 8, '10': 'alwaysRun'},
   ],
 };
 
 /// Descriptor for `PrecompiledObject`. Decode as a `google.protobuf.DescriptorProto`.
-final $typed_data.Uint8List precompiledObjectDescriptor = $convert.base64Decode('ChFQcmVjb21waWxlZE9iamVjdBIdCgpjbG91ZF9wYXRoGAEgASgJUgljbG91ZFBhdGgSEgoEbmFtZRgCIAEoCVIEbmFtZRIgCgtkZXNjcmlwdGlvbhgDIAEoCVILZGVzY3JpcHRpb24SMQoEdHlwZRgEIAEoDjIdLmFwaS52MS5QcmVjb21waWxlZE9iamVjdFR5cGVSBHR5cGUSKQoQcGlwZWxpbmVfb3B0aW9ucxgFIAEoCVIPcGlwZWxpbmVPcHRpb25zEhIKBGxpbmsYBiABKAlSBGxpbmsSHAoJbXVsdGlmaWxlGAcgASgIUgltdWx0aWZpbGUSIQoMY29udGV4dF9saW5lGAggASgFUgtjb250ZXh0TGluZRInCg9kZWZhdWx0X2V4YW1wbGUYCSABKAhSDmRlZmF1bHRFeGFtcGxlEh0KA3NkaxgKIAEoDjILLmFwaS52MS5TZGtSA3NkaxIyCgpjb21wbGV4aXR5GAsgASgOMhIuYXBpLnYxLkNvbXBsZXhpdHlSCmNvbXBsZXhpdHkSEgoEdGFncxgMIAMoCVIEdGFncxIrCghkYXRhc2V0cxgNIAMoCzIPLmFwaS52MS5EYXRhc2V0UghkYXRhc2V0cxIXCgd1cmxfdmNzGA4gASgJUgZ1cmxWY3MSIQoMdXJsX25vdGVib29rGA8gASgJUgt1cmxOb3RlYm9vaw==');
+final $typed_data.Uint8List precompiledObjectDescriptor = $convert.base64Decode('ChFQcmVjb21waWxlZE9iamVjdBIdCgpjbG91ZF9wYXRoGAEgASgJUgljbG91ZFBhdGgSEgoEbmFtZRgCIAEoCVIEbmFtZRIgCgtkZXNjcmlwdGlvbhgDIAEoCVILZGVzY3JpcHRpb24SMQoEdHlwZRgEIAEoDjIdLmFwaS52MS5QcmVjb21waWxlZE9iamVjdFR5cGVSBHR5cGUSKQoQcGlwZWxpbmVfb3B0aW9ucxgFIAEoCVIPcGlwZWxpbmVPcHRpb25zEhIKBGxpbmsYBiABKAlSBGxpbmsSHAoJbXVsdGlmaWxlGAcgASgIUgltdWx0aWZpbGUSIQoMY29udGV4dF9saW5lGAggASgFUgtjb250ZXh0TGluZRInCg9kZWZhdWx0X2V4YW1wbGUYCSABKAhSDmRlZmF1bHRFeGFtcGxlEh0KA3NkaxgKIAEoDjILLmFwaS52MS5TZGtSA3NkaxIyCgpjb21wbGV4aXR5GAsgASgOMhIuYXBpLnYxLkNvbXBsZXhpdHlSCmNvbXBsZXhpdHkSEgoEdGFncxgMIAMoCVIEdGFncxIrCghkYXRhc2V0cxgNIAMoCzIPLmFwaS52MS5EYXRhc2V0UghkYXRhc2V0cxIXCgd1cmxfdmNzGA4gASgJUgZ1cmxWY3MSIQoMdXJsX25vdGVib29rGA8gASgJUgt1cmxOb3RlYm9vaxIdCgphbHdheXNfcnVuGBAgASgIUglhbHdheXNSdW4=');
 @$core.Deprecated('Use categoriesDescriptor instead')
 const Categories$json = const {
   '1': 'Categories',

diff --git a/playground/frontend/playground_components/lib/src/cache/example_cache.dart b/playground/frontend/playground_components/lib/src/cache/example_cache.dart
@@ -95,27 +95,31 @@ class ExampleCache extends ChangeNotifier {
     );
   }
 
-  Future<String> _getPrecompiledObjectOutput(String path, Sdk sdk) {
+  Future<String?> _getPrecompiledObjectOutput(ExampleBase example) async {
+    if (example.alwaysRun) {
+      return null;
+    }
+
     return _exampleRepository.getPrecompiledObjectOutput(
-      GetPrecompiledObjectRequest(path: path, sdk: sdk),
+      GetPrecompiledObjectRequest(path: example.path, sdk: example.sdk),
     );
   }
 
-  Future<List<SnippetFile>> _getPrecompiledObjectCode(String path, Sdk sdk) {
+  Future<List<SnippetFile>> _getPrecompiledObjectCode(ExampleBase example) {
     return _exampleRepository.getPrecompiledObjectCode(
-      GetPrecompiledObjectRequest(path: path, sdk: sdk),
+      GetPrecompiledObjectRequest(path: example.path, sdk: example.sdk),
     );
   }
 
-  Future<String> _getPrecompiledObjectLogs(String path, Sdk sdk) {
+  Future<String> _getPrecompiledObjectLogs(ExampleBase example) {
     return _exampleRepository.getPrecompiledObjectLogs(
-      GetPrecompiledObjectRequest(path: path, sdk: sdk),
+      GetPrecompiledObjectRequest(path: example.path, sdk: example.sdk),
     );
   }
 
-  Future<String> _getPrecompiledObjectGraph(String id, Sdk sdk) {
+  Future<String> _getPrecompiledObjectGraph(ExampleBase example) {
     return _exampleRepository.getPrecompiledObjectGraph(
-      GetPrecompiledObjectRequest(path: id, sdk: sdk),
+      GetPrecompiledObjectRequest(path: example.path, sdk: example.sdk),
     );
   }
 
@@ -168,33 +172,33 @@ class ExampleCache extends ChangeNotifier {
     if (example.name == 'MinimalWordCount' &&
         (example.sdk == Sdk.go || example.sdk == Sdk.scio)) {
       final exampleData = await Future.wait([
-        _getPrecompiledObjectCode(example.path, example.sdk),
-        _getPrecompiledObjectOutput(example.path, example.sdk),
-        _getPrecompiledObjectLogs(example.path, example.sdk),
+        _getPrecompiledObjectCode(example),
+        _getPrecompiledObjectOutput(example),
+        _getPrecompiledObjectLogs(example),
       ]);
 
       return Example.fromBase(
         example,
-        files: exampleData[0] as List<SnippetFile>,
-        outputs: exampleData[1] as String,
-        logs: exampleData[2] as String,
+        files: exampleData[0]! as List<SnippetFile>,
+        outputs: exampleData[1] as String?,
+        logs: exampleData[2]! as String,
       );
     }
 
     // TODO(alexeyinkin): Load in a single request, https://github.com/apache/beam/issues/24305
     final exampleData = await Future.wait([
-      _getPrecompiledObjectCode(example.path, example.sdk),
-      _getPrecompiledObjectOutput(example.path, example.sdk),
-      _getPrecompiledObjectLogs(example.path, example.sdk),
-      _getPrecompiledObjectGraph(example.path, example.sdk)
+      _getPrecompiledObjectCode(example),
+      _getPrecompiledObjectOutput(example),
+      _getPrecompiledObjectLogs(example),
+      _getPrecompiledObjectGraph(example)
     ]);
 
     return Example.fromBase(
       example,
-      files: exampleData[0] as List<SnippetFile>,
-      outputs: exampleData[1] as String,
-      logs: exampleData[2] as String,
-      graph: exampleData[3] as String,
+      files: exampleData[0]! as List<SnippetFile>,
+      outputs: exampleData[1] as String?,
+      logs: exampleData[2]! as String,
+      graph: exampleData[3]! as String,
     );
   }
 

diff --git a/...end/playground_components/lib/src/controllers/example_loaders/content_example_loader.dart b/...end/playground_components/lib/src/controllers/example_loaders/content_example_loader.dart
@@ -45,6 +45,7 @@ class ContentExampleLoader extends ExampleLoader {
         files: descriptor.files,
         name: descriptor.name ?? 'Untitled Example',
         path: '',
+        pipelineOptions: descriptor.pipelineOptions,
         sdk: descriptor.sdk,
         type: ExampleType.example,
         viewOptions: descriptor.viewOptions,

diff --git a/...ground/frontend/playground_components/lib/src/controllers/snippet_editing_controller.dart b/...ground/frontend/playground_components/lib/src/controllers/snippet_editing_controller.dart
@@ -154,6 +154,7 @@ class SnippetEditingController extends ChangeNotifier {
       complexity: example.complexity,
       files: getFiles(),
       name: example.name,
+      pipelineOptions: _pipelineOptions,
       sdk: sdk,
     );
   }

diff --git a/playground/frontend/playground_components/lib/src/models/example_base.dart b/playground/frontend/playground_components/lib/src/models/example_base.dart
@@ -50,6 +50,7 @@ extension ExampleTypeToString on ExampleType {
 /// and other large fields.
 /// These objects are fetched as lists from [ExampleRepository].
 class ExampleBase with Comparable<ExampleBase>, EquatableMixin {
+  final bool alwaysRun;
   final Complexity? complexity;
 
   /// Index of the line to focus, 1-based.
@@ -72,6 +73,7 @@ class ExampleBase with Comparable<ExampleBase>, EquatableMixin {
     required this.path,
     required this.sdk,
     required this.type,
+    this.alwaysRun = false,
     this.complexity,
     this.contextLine = 1,
     this.datasets = const [],

diff --git a/...onents/lib/src/models/example_loading_descriptors/content_example_loading_descriptor.dart b/...onents/lib/src/models/example_loading_descriptors/content_example_loading_descriptor.dart
@@ -26,12 +26,14 @@ import 'example_loading_descriptor.dart';
 
 /// Fully contains an example data to be loaded.
 class ContentExampleLoadingDescriptor extends ExampleLoadingDescriptor {
+  final Complexity? complexity;
+
   final List<SnippetFile> files;
 
   /// The name of the example, if any, to show in the dropdown.
   final String? name;
 
-  final Complexity? complexity;
+  final String pipelineOptions;
 
   final Sdk sdk;
 
@@ -40,6 +42,7 @@ class ContentExampleLoadingDescriptor extends ExampleLoadingDescriptor {
     required this.sdk,
     this.complexity,
     this.name,
+    this.pipelineOptions = '',
     super.viewOptions,
   });
 
@@ -55,12 +58,13 @@ class ContentExampleLoadingDescriptor extends ExampleLoadingDescriptor {
     }
 
     return ContentExampleLoadingDescriptor(
+      complexity: Complexity.fromString(map['complexity']),
       files: files
           .map((file) => SnippetFile.fromJson(file as Map<String, dynamic>))
           .toList(growable: false),
       name: map['name']?.toString(),
+      pipelineOptions: map['pipelineOptions'] ?? '',
       sdk: sdk,
-      complexity: Complexity.fromString(map['complexity']),
       viewOptions: ExampleViewOptions.fromShortMap(map),
     );
   }
@@ -87,6 +91,7 @@ class ContentExampleLoadingDescriptor extends ExampleLoadingDescriptor {
         complexity,
         files,
         name,
+        pipelineOptions,
         sdk.id,
         viewOptions,
       ];
@@ -97,6 +102,7 @@ class ContentExampleLoadingDescriptor extends ExampleLoadingDescriptor {
         complexity: complexity,
         files: files,
         name: name,
+        pipelineOptions: pipelineOptions,
         sdk: sdk,
       );
 
@@ -105,6 +111,7 @@ class ContentExampleLoadingDescriptor extends ExampleLoadingDescriptor {
         'complexity': complexity?.name,
         'files': files.map((e) => e.toJson()).toList(growable: false),
         'name': name,
+        'pipelineOptions': pipelineOptions,
         'sdk': sdk.id,
         ...viewOptions.toShortMap(),
       };

diff --git a/...ontend/playground_components/lib/src/repositories/example_client/grpc_example_client.dart b/...ontend/playground_components/lib/src/repositories/example_client/grpc_example_client.dart
@@ -335,6 +335,7 @@ class GrpcExampleClient implements ExampleClient {
 
   ExampleBase _toExampleModel(Sdk sdk, grpc.PrecompiledObject example) {
     return ExampleBase(
+      alwaysRun: example.alwaysRun,
       complexity: example.complexity.model,
       contextLine: example.contextLine,
       description: example.description,

diff --git a/.../test/src/models/example_loading_descriptors/content_example_loading_descriptor_test.dart b/.../test/src/models/example_loading_descriptors/content_example_loading_descriptor_test.dart
@@ -39,6 +39,7 @@ void main() {
       complexity: Complexity.advanced,
       files: [SnippetFile(content: 'abc', isMain: true)],
       name: 'name',
+      pipelineOptions: '--name=value',
       sdk: Sdk.go,
       viewOptions: viewOptions,
     );
@@ -57,6 +58,7 @@ void main() {
           complexity: descriptor.complexity,
           files: descriptor.files,
           name: descriptor.name,
+          pipelineOptions: '--name=value',
           sdk: descriptor.sdk,
         ),
       );