diff --git a/README.md b/README.md index d67740f..02be5d4 100644 --- a/README.md +++ b/README.md @@ -34,14 +34,19 @@ Smash some code into your keyboard, click submit, and watch as Chainless magical - Ethereum - Apparatus +Support for other chains will be added over time. If you'd like to request one, feel free to file an [Issue](https://github.com/SeanCheatham/chainless/issues). Bonus points if you provide some reference material to assist with the integration. Bonus bonus points if you file a PR with an implementation! + ## Supported Languages **Temporary Functions** - JavaScript (GraalVM) +- Python (GraalVM) **Persistent Functions** - JavaScript/TypeScript (NodeJS) - Java/Scala/Kotlin (JVM) +Support for other languages will be added over time. If you'd like to request one, feel free to file an [Issue](https://github.com/SeanCheatham/chainless/issues). Bonus points if you provide some reference material to assist with the integration. Bonus bonus points if you file a PR with an implementation! + ## FAQ - Is Chainless AI-native? - No. diff --git a/backend/core/src/main/scala/chainless/ApiServer.scala b/backend/core/src/main/scala/chainless/ApiServer.scala index 7c60974..c8af68a 100644 --- a/backend/core/src/main/scala/chainless/ApiServer.scala +++ b/backend/core/src/main/scala/chainless/ApiServer.scala @@ -225,7 +225,7 @@ class ApiServer( .map(request => Response( body = operator - .retroact(request.code)(request.timestampMs, request.chains) + .retroact(request.code, request.language)(request.timestampMs, request.chains) .map(_.asJson) .map(_.noSpaces) .mergeHaltL(keepAliveTick(2.seconds)) @@ -245,6 +245,7 @@ class ApiServer( body = operator .live( request.code, + request.language, FunctionState(request.chainStates.getOrElse(Map.empty), request.state.getOrElse(Json.Null)) )(request.chains) .map(_.asJson) diff --git a/backend/core/src/main/scala/chainless/ChainlessMain.scala b/backend/core/src/main/scala/chainless/ChainlessMain.scala index 72b0d8b..31a1e61 100644 --- a/backend/core/src/main/scala/chainless/ChainlessMain.scala +++ b/backend/core/src/main/scala/chainless/ChainlessMain.scala @@ -75,39 +75,40 @@ object ChainlessMain extends ResourceApp.Forever { functionInvocationsDb: FunctionInvocationsDb[F], blocksDb: BlocksDb[F] ) = - Ref - .of[F, Boolean](false) - .toResource - .flatMap(canceledRef => - NonEmptyChain - .fromChainUnsafe(Chain.fromSeq(List.tabulate(args.runnerCount)(i => i + args.runnerApiBindPortStart))) - .traverse(port => - Files[F] - .tempDirectory(Some(Path(args.sharedTmpDir)), port.toString, None) - .flatMap(localCodeCache => - DockerDriver.make[F]( - s"http://chainless:$port", - objectStore, - localCodeCache + Files[F].createDirectories(Path(args.sharedTmpDir)).toResource >> + Ref + .of[F, Boolean](false) + .toResource + .flatMap(canceledRef => + NonEmptyChain + .fromChainUnsafe(Chain.fromSeq(List.tabulate(args.runnerCount)(i => i + args.runnerApiBindPortStart))) + .traverse(port => + Files[F] + .tempDirectory(Some(Path(args.sharedTmpDir)), port.toString, None) + .flatMap(localCodeCache => + DockerDriver.make[F]( + s"http://chainless:$port", + objectStore, + localCodeCache + ) ) - ) - .flatMap(dockerDriver => - JobProcessor.make[F]( - dockerDriver, - functionsDb, - functionInvocationsDb, - blocksDb, - objectStore, - canceledRef + .flatMap(dockerDriver => + JobProcessor.make[F]( + dockerDriver, + functionsDb, + functionInvocationsDb, + blocksDb, + objectStore, + canceledRef + ) ) - ) - .flatTap(jobProcessor => - new RunnerHttpServer(jobProcessor.nextTask, jobProcessor.completeTask).serve("0.0.0.0", port) - ) - ) - .flatMap(MultiJobProcessor.make[F]) - .flatTap(_ => Resource.onFinalize(canceledRef.set(true))) - ) + .flatTap(jobProcessor => + new RunnerHttpServer(jobProcessor.nextTask, jobProcessor.completeTask).serve("0.0.0.0", port) + ) + ) + .flatMap(MultiJobProcessor.make[F]) + .flatTap(_ => Resource.onFinalize(canceledRef.set(true))) + ) } @AppName("Chainless") diff --git a/backend/core/src/main/scala/chainless/models/package.scala b/backend/core/src/main/scala/chainless/models/package.scala index 2d547a7..525816b 100644 --- a/backend/core/src/main/scala/chainless/models/package.scala +++ b/backend/core/src/main/scala/chainless/models/package.scala @@ -12,10 +12,11 @@ import scala.concurrent.duration.* package object models: case class InitRequest(code: String, config: Json) - case class RetroactRequest(code: String, timestampMs: Long, chains: NonEmptyChain[Chain]) + case class RetroactRequest(code: String, timestampMs: Long, chains: NonEmptyChain[Chain], language: String = "js") case class StreamRequest( code: String, + language: String = "js", chainStates: Option[Map[String, String]], state: Option[Json], chains: NonEmptyChain[Chain] diff --git a/backend/core/src/main/scala/chainless/runner/temporary/Runner.scala b/backend/core/src/main/scala/chainless/runner/temporary/Runner.scala index 0ee46ab..aba66a4 100644 --- a/backend/core/src/main/scala/chainless/runner/temporary/Runner.scala +++ b/backend/core/src/main/scala/chainless/runner/temporary/Runner.scala @@ -7,25 +7,25 @@ import cats.NonEmptyParallel import chainless.models.{*, given} import io.circe.syntax.* import io.circe.{Json, JsonNumber, JsonObject} -import org.graalvm.polyglot.proxy.{ProxyArray, ProxyObject} +import org.graalvm.polyglot.proxy.* import org.graalvm.polyglot.{Context, Value} import java.util.concurrent.Executors +import scala.annotation.tailrec import scala.concurrent.ExecutionContext import scala.jdk.CollectionConverters.* /** A running instance of a temporary function. Applies each block to the current function. - * @tparam F */ trait Runner[F[_]]: def applyBlock(stateWithChains: FunctionState, blockWithChain: BlockWithChain): F[FunctionState] object LocalGraalRunner: import GraalSupport.* - def make[F[_]: Async: NonEmptyParallel](code: String): Resource[F, Runner[F]] = + def make[F[_]: Async: NonEmptyParallel](code: String, language: String): Resource[F, Runner[F]] = GraalSupport .makeContext[F] - .evalMap((ec, context) => ec.evalSync(context.eval("js", code)).map((ec, context, _))) + .evalMap((ec, context) => ec.evalSync(context.eval(language, code)).map((ec, context, _))) .map((ec, context, compiled) => new Runner[F]: given Context = context @@ -39,9 +39,17 @@ object LocalGraalRunner: .guarantee(Async[F].cede) .flatMap((stateWithChainsJson, blockWithChainJson) => ec.evalSync { - val result = compiled.execute(stateWithChainsJson.asValue, blockWithChainJson.asValue) - val json = result.asJson - json + if (language == "js") { + val result = compiled.execute(stateWithChainsJson.asValue, blockWithChainJson.asValue) + val json = result.asJson + json + } else { + val result = context.getPolyglotBindings + .getMember("apply_block") + .execute(stateWithChainsJson.asValue, blockWithChainJson.asValue) + val json = result.asJson + json + } } ) .guarantee(Async[F].cede) @@ -70,8 +78,8 @@ object GraalSupport: ) .flatMap(executor => Resource - .make(executor.eval(Sync[F].delay(Context.create())))(context => - executor.eval(Sync[F].blocking(context.close())) + .make(executor.eval(Sync[F].delay(Context.newBuilder("js", "python").allowAllAccess(true).build())))( + context => executor.eval(Sync[F].blocking(context.close())) ) .tupleLeft(executor) ) @@ -119,11 +127,11 @@ object GraalSupport: } def onObject(value: JsonObject): Value = { - val map = new java.util.HashMap[String, AnyRef](value.size) + val map = new java.util.HashMap[Object, Object](value.size) value.toMap.foreach { case (key, value) => map.put(key, value.asValue) } - context.asValue(ProxyObject.fromMap(map)) + context.asValue(ProxyHashMap.from(map)) } } @@ -144,19 +152,19 @@ object GraalSupport: k.asString() -> v.asJson }.toSeq* ) - else if (value.hasMembers) + else if (value.hasMembers) { Json.obj( - value.asScalaMapIterator.map { case (k, v) => - k.asString() -> v.asJson - }.toSeq* + value.getMemberKeys.asScala.map(key => key -> value.getMember(key).asJson).toSeq* ) - else throw new MatchError(value) + } else throw new MatchError(value) + @tailrec def asScalaIterator(using context: Context): Iterator[Value] = if (value.isIterator) { Iterator.unfold(value)(i => Option.when(i.hasIteratorNextElement)(i.getIteratorNextElement -> i)) } else value.getIterator.asScalaIterator + @tailrec def asScalaMapIterator(using context: Context): Iterator[(Value, Value)] = if (value.isIterator) { Iterator.unfold(value)(i => @@ -166,8 +174,6 @@ object GraalSupport: (arr.getArrayElement(0) -> arr.getArrayElement(1)) -> i } ) - } else if (value.hasMembers) - value.getMemberKeys.asScala.iterator.map(key => context.asValue(key) -> value.getMember(key)) - else value.getHashEntriesIterator.asScalaMapIterator + } else value.getHashEntriesIterator.asScalaMapIterator extension (json: Json) def asValue(using context: Context): Value = json.foldWith(jsonFolder) diff --git a/backend/core/src/main/scala/chainless/runner/temporary/RunnerOperator.scala b/backend/core/src/main/scala/chainless/runner/temporary/RunnerOperator.scala index 255273f..42b3a07 100644 --- a/backend/core/src/main/scala/chainless/runner/temporary/RunnerOperator.scala +++ b/backend/core/src/main/scala/chainless/runner/temporary/RunnerOperator.scala @@ -35,10 +35,11 @@ class RunnerOperator[F[_]: Async: NonEmptyParallel]( given Logger[F] = Slf4jLogger.getLoggerFromName("Operator") def retroact( - code: String + code: String, + language: String )(timestampMs: Long, chains: NonEmptyChain[Chain]): Stream[F, FunctionState] = Stream - .resource(LocalGraalRunner.make[F](code)) + .resource(LocalGraalRunner.make[F](code, language)) .flatMap(runner => blocksDb .blocksAfterTimestamp(chains)(timestampMs) @@ -52,11 +53,11 @@ class RunnerOperator[F[_]: Async: NonEmptyParallel]( .map(_._2) ) - def live(code: String, stateWithChains: FunctionState)( + def live(code: String, language: String, stateWithChains: FunctionState)( chains: NonEmptyChain[Chain] ): Stream[F, FunctionState] = Stream - .resource(LocalGraalRunner.make[F](code)) + .resource(LocalGraalRunner.make[F](code, language)) .flatMap(runner => newBlocks .filter(meta => chains.contains(meta.chain)) diff --git a/backend/project/Dependencies.scala b/backend/project/Dependencies.scala index bab228f..c5abdc6 100644 --- a/backend/project/Dependencies.scala +++ b/backend/project/Dependencies.scala @@ -45,7 +45,8 @@ object Dependencies { val graalVM = Seq( "org.graalvm.polyglot" % "polyglot" % "24.0.2", - "org.graalvm.polyglot" % "js-community" % "24.0.2" + "org.graalvm.polyglot" % "js-community" % "24.0.2", + "org.graalvm.polyglot" % "python-community" % "24.0.2" ) val http4s = Seq( diff --git a/docs/docs/temporary-functions.md b/docs/docs/temporary-functions.md index f0da090..3e78ba1 100644 --- a/docs/docs/temporary-functions.md +++ b/docs/docs/temporary-functions.md @@ -6,10 +6,11 @@ These functions only runs while the connection is open. Once closed, the functi ### Prepare Open the [Chainless App](http://localhost:42069). Then, click the button in the bottom-right corner to expand the menu. Select "Create Temporary Function". -At this page, you can write your function code as a single JavaScript file. Temporary functions may not include additional dependencies, so they are limited in functionality. Edit the code box to implement your function. +At this page, you can write your function code as a single JavaScript or Python file. Temporary functions may not include additional dependencies, so they are limited in functionality. Edit the code box to implement your function. ### Code The code is generally structured in the following manner: +**JS** ```js (function(functionState, blockWithMeta) { let state = { ...functionState.state } @@ -17,6 +18,24 @@ The code is generally structured in the following manner: return state }) ``` +*Note: The entire function must be wrapped in (parenthesis).* + +**Python** +```python +import polyglot +@polyglot.export_value +def apply_block(stateWithChains, blockWithMeta): + if(stateWithChains["state"] is not None): + state = stateWithChains["state"] + # Modify state + return state + else: + state = {} + # Initialize state + return state +``` +*Note: The "polyglot" statements are required* +*Note: Python functionality is in early-preview* This function accepts a function state and a new block, and produces a new state. It is invoked over-and-over again by the backend as new blocks arrive. @@ -24,8 +43,6 @@ The `functionState` object contains `.state` and `.chainStates` fields. `.state` The `blockWithMeta` object contains `.meta` and `.block` fields. `.meta` includes information like `.chain`, `.blockId`, and `.height`. `.block` contains the raw JSON-encoded data of the block. -The entire function must be wrapped in (parenthesis). - ### Historical Blocks You can optionally select a `Start Time` from which blocks should be retroactively applied. For example, you can instruct the function to apply all blocks from yesterday before applying new blocks. diff --git a/frontend/lib/http/api_client.dart b/frontend/lib/http/api_client.dart index 411a49c..9ded93c 100644 --- a/frontend/lib/http/api_client.dart +++ b/frontend/lib/http/api_client.dart @@ -1,6 +1,7 @@ import 'dart:convert'; import 'dart:async'; import 'package:chainless_frontend/models/models.dart'; +import 'package:flutter/foundation.dart'; import 'package:http/http.dart' as http; import 'package:rxdart/rxdart.dart'; import './http_client.dart'; @@ -8,7 +9,8 @@ import './http_client.dart'; class PublicApiClient { final String baseAddress; - static const defaultBaseAddress = "/api"; + static const defaultBaseAddress = + kDebugMode ? "http://localhost:42069/api" : "/api"; PublicApiClient({this.baseAddress = defaultBaseAddress}); @@ -139,11 +141,12 @@ class PublicApiClient { } Stream retroact( - String code, DateTime timestamp, List chains) { + String code, String language, DateTime timestamp, List chains) { final client = makeHttpClient(); Future>> call() async { final body = { "code": code, + "language": language, "timestampMs": timestamp.millisecondsSinceEpoch, "chains": chains, }; @@ -175,12 +178,13 @@ class PublicApiClient { .doOnError((_, __) => client.close()); } - Stream streamed( - String code, FunctionState stateWithChains, List chains) { + Stream streamed(String code, String language, + FunctionState stateWithChains, List chains) { final client = makeHttpClient(); Future>> call() async { final body = { "code": code, + "language": language, "chainStates": stateWithChains.chainStates, "state": stateWithChains.state, "chains": chains, diff --git a/frontend/lib/temporary_function_page.dart b/frontend/lib/temporary_function_page.dart index 767c028..b4bc031 100644 --- a/frontend/lib/temporary_function_page.dart +++ b/frontend/lib/temporary_function_page.dart @@ -7,6 +7,7 @@ import 'package:chainless_frontend/ui_utils.dart'; import 'package:chainless_frontend/widgets/chain_selection_dropdown.dart'; import 'package:chainless_frontend/widgets/gradient_background.dart'; import 'package:flutter/material.dart'; +import 'package:multi_dropdown/multiselect_dropdown.dart'; import 'package:omni_datetime_picker/omni_datetime_picker.dart'; import 'package:code_editor/code_editor.dart'; import 'package:provider/provider.dart'; @@ -19,7 +20,8 @@ class TemporaryFunctionPage extends StatefulWidget { } class _TemporaryFunctionPageState extends State { - String code = _code; + String code = _jsCode; + String language = "js"; DateTime? initTime; late EditorModel model; List chains = ["bitcoin", "ethereum", "apparatus"]; @@ -27,12 +29,7 @@ class _TemporaryFunctionPageState extends State { @override void initState() { super.initState(); - model = EditorModel( - styleOptions: codeEditorStyle(500), - files: [ - FileEditor(name: "Temporary Function", language: "js", code: code) - ], - ); + model = _buildModel; } @override @@ -69,6 +66,8 @@ class _TemporaryFunctionPageState extends State { fontWeight: FontWeight.bold, )).pad16, divider, + languageField(), + divider, chainsField(), divider, timePickerField(), @@ -85,10 +84,18 @@ class _TemporaryFunctionPageState extends State { static const divider = Divider(thickness: 0, height: 24, color: Colors.transparent); + EditorModel get _buildModel => EditorModel( + styleOptions: codeEditorStyle(500), + files: [ + FileEditor(name: "Temporary Function", language: language, code: code) + ], + ); + Widget editor() { return CodeEditor( + key: UniqueKey(), model: model, - formatters: const ["js"], + formatters: const ["js", "python"], onSubmit: (_, c) { setState(() { code = c; @@ -110,6 +117,37 @@ class _TemporaryFunctionPageState extends State { ); } + Widget languageField() => FieldWithHeader( + name: "Language", + required: true, + tooltip: "The programming language used to develop your function.", + child: MultiSelectDropDown( + onOptionSelected: (selectedOptions) { + if (selectedOptions.isNotEmpty) { + final value = selectedOptions.first.value; + if (value != null) { + final newLanguage = selectedOptions.first.value ?? "js"; + final previousLanguage = language; + final codeWasDefault = _codeIsDefault(previousLanguage, code); + setState(() { + language = newLanguage; + if (codeWasDefault) { + code = newLanguage == "js" ? _jsCode : _pythonCode; + model = _buildModel; + } + }); + } + } + }, + options: [ + for (final c in selectableLanguages) ValueItem(label: c, value: c) + ], + selectionType: SelectionType.single, + selectedOptionIcon: const Icon(Icons.check_circle), + selectedOptions: [ValueItem(label: language, value: language)], + ), + ); + Widget timePickerField() { final icon = TextButton.icon( onPressed: () => showOmniDateTimePicker( @@ -143,6 +181,7 @@ class _TemporaryFunctionPageState extends State { MaterialPageRoute( builder: (context) => StateViewer( code: code, + language: language, initTime: initTime, chains: chains, ), @@ -152,16 +191,28 @@ class _TemporaryFunctionPageState extends State { label: const Text("Run"), ); } + + bool _codeIsDefault(String language, String code) { + if (language == "js") { + return code == _jsCode; + } else if (language == "python") { + return code == _pythonCode; + } else { + return false; + } + } } class StateViewer extends StatelessWidget { final String code; + final String language; final DateTime? initTime; final List chains; const StateViewer( {super.key, required this.code, + required this.language, required this.initTime, required this.chains}); @@ -186,14 +237,16 @@ class StateViewer extends StatelessWidget { Stream stream(PublicApiClient client) async* { late FunctionState retroacted; if (initTime != null) { - await for (final next in client.retroact(code, initTime!, chains)) { + await for (final next + in client.retroact(code, language, initTime!, chains)) { yield next; retroacted = next; } } else { retroacted = FunctionState(chainStates: {}, state: null); } - await for (final next in client.streamed(code, retroacted, chains)) { + await for (final next + in client.streamed(code, language, retroacted, chains)) { yield next; } } @@ -204,7 +257,7 @@ class StateViewer extends StatelessWidget { chainStates: functionState.chainStates, state: functionState.state, name: "Temporary Function", - language: "js", + language: language, chains: chains, error: null, initialized: true, @@ -217,7 +270,7 @@ class StateViewer extends StatelessWidget { ); } -const _code = """ +const _jsCode = """ // A function which counts blocks, grouped by chain // A temporary function accepts two arguments: the current state and a new block (function(functionState, blockWithMeta) { @@ -238,3 +291,18 @@ const _code = """ return state; }); """; + +const _pythonCode = """ +import polyglot +@polyglot.export_value +def apply_block(stateWithChains, blockWithMeta): + state = stateWithChains["state"] or {} + chain = blockWithMeta["meta"]["chain"] + if chain in state: + state[chain] = state[chain] + 1 + else: + state[chain] = 1 + return state +"""; + +const selectableLanguages = ["js", "python"]; diff --git a/frontend/lib/widgets/function_view.dart b/frontend/lib/widgets/function_view.dart index 3ab04e9..0c49733 100644 --- a/frontend/lib/widgets/function_view.dart +++ b/frontend/lib/widgets/function_view.dart @@ -317,6 +317,10 @@ Widget languageIcon(String language, {Color? color, double? size}) { return Tooltip( message: "JVM", child: FaIcon(FontAwesomeIcons.java, color: color, size: size)); + case "python": + return Tooltip( + message: "Python", + child: FaIcon(FontAwesomeIcons.python, color: color, size: size)); default: return Tooltip( message: "Unknown",