From c65c68e4c2aae7a190730c91e9a394a7fcd8ca09 Mon Sep 17 00:00:00 2001 From: Griatch Date: Sat, 27 Mar 2021 19:20:21 +0100 Subject: [PATCH] Further cleanup and refactoring --- docs/source/Components/FuncParser.md | 455 +++++++++++++++---------- evennia/objects/objects.py | 2 +- evennia/utils/funcparser.py | 191 +++++------ evennia/utils/tests/test_funcparser.py | 26 +- evennia/utils/tests/test_utils.py | 41 +++ evennia/utils/utils.py | 103 ++++++ 6 files changed, 526 insertions(+), 292 deletions(-) diff --git a/docs/source/Components/FuncParser.md b/docs/source/Components/FuncParser.md index ebdd073441..d0d19329ff 100644 --- a/docs/source/Components/FuncParser.md +++ b/docs/source/Components/FuncParser.md @@ -1,6 +1,9 @@ # The Inline Function Parser -The [FuncParser](api:evennia.utils.funcparser.FuncParser) extracts and executes 'inline functions' +## Introduction + +The [FuncParser](api:evennia.utils.funcparser#evennia.utils.funcparser.FuncParser) extracts and executes +'inline functions' embedded in a string on the form `$funcname(args, kwargs)`. Under the hood, this will lead to a call to a Python function you control. The inline function call will be replaced by the return from the function. @@ -8,105 +11,60 @@ the return from the function. ```python from evennia.utils.funcparser import FuncParser -def _square(*args, **kwargs): +def _square_callable(*args, **kwargs): """This will be callable as $square(number) in string""" return float(args[0]) ** 2 -parser = FuncParser({"square": _square}) +parser = FuncParser({"square": _square_callable}) + +``` +Next, just pass a string into the parser, optionally containing `$func(...)` markers: + +```python parser.parse("We have that 4 x 4 is $square(4).") "We have that 4 x 4 is 16." ``` -Normally the return is always converted to a string but you can also retrieve other data types -from the function calls: +Normally the return is always converted to a string but you can also get the actual data type from the call: ```python parser.parse_to_any("$square(4)") 16 ``` -To show a `$func()` verbatim in your code without parsing it, escape it as either `$$func()` or `\$func()`. - -The point of inline-parsed functions is that they allow users to call dynamic code without giving -regular users full access to Python. You can supply any python function to process the users' input. - -Here are some more examples: - - "Let's meet at our guild hall. Here's how you get here: $route(Warrior's Guild)." - -This can be parsed when sending messages, the users's current session passed into the callable. Assuming the -game used a grid system and some path-finding mechanism, this would calculate the route to the guild -individually for each recipient, such as: - - player1: "Let's meet at our guild hall. Here's how you get here: north,west,north,north. - player2: "Let's meet at our guild hall. Here's how you get here: south,east. - player3: "Let's meet at our guild hall. Here's how you get here: south,south,south,east. - -It can be used (by user or developer) to implement _Actor stance emoting_ (2nd person) so people see -different variations depending on who they are (the [RPSystem contrib](../Contribs/Contrib-Overview) does -this in a different way for _Director stance_): - - sendstr = "$me() $inflect(look) at the $obj(garden)." - - I see: "You look at the Splendid Green Garden." - others see: "Anna looks at the Splendid Green Garden." - -... embedded dice rolls ... - - "I make a sweeping attack and roll $roll(2d6)!" - "I make a sweeping attack and roll 8 (3+5 on 2d6)!" - -Function calls can also be nested. Here's an example of inline formatting - - "This is a $fmt('-' * 20, $clr(r, red text)!, '-' * 20") - "This is a --------------------red text!--------------------" +To show a `$func()` verbatim in your code without parsing it, escape it as either `$$func()` or `\$func()`: -```important:: - The inline-function parser is not intended as a 'softcode' programming language. It does not - have things like loops and conditionals, for example. While you could in principle extend it to - do very advanced things and allow builders a lot of power, all-out coding is something - Evennia expects you to do in a proper text editor, outside of the game, not from inside it. +```python +parser.parse("This is an escaped $$square(4) and so is this \$square(3)") +"This is an escaped $square(4) and so is this $square(3)" ``` -## Standard uses of parser -Out of the box, Evennia applies the parser in two situations: +## Uses in default Evennia -### Inlinefunc parsing +The FuncParser can be applied to any string. Out of the box it's applied in a few situations: -This is inactive by default. When active, Evennia will run the parser on _every outgoing string_ -from a character, making the current [Session](./Sessions) available to every callable. This allows for a single string -to appear differently to different users (see the example of `$route()` or `$me()`) above). +- _Outgoing messages_. All messages sent from the server is processed through FuncParser and every + callable is provided the [Session](Sessions) of the object receiving the message. This potentially + allows a message to be modified on the fly to look different for different recipients. +- _Prototype values_. A [Prototype](Prototypes) dict's values are run through the parser such that every + callable gets a reference to the rest of the prototype. In the Prototype ORM, this would allow builders + to safely call functions to set non-string values to prototype values, get random values, reference + other fields of the prototype, and more. +- _Actor-stance in messages to others_. In the + [Object.msg_contents](api:evennia.objects.objects#objects.objects.DefaultObject.msg_contents) method, + the outgoing string is parsed for special `$You()` and `$conj()` callables to decide if a given recipient + should see "You" or the character's name. -To turn on this parsing, set `INLINEFUNC_ENABLED=True` in your settings file. You can add more callables in -`mygame/server/conf/inlinefuncs.py` and expand the list `INLINEFUNC_MODULES` with paths to modules containing -callables. - -These are some example callables distributed with Evennia for inlinefunc-use. +```important:: + The inline-function parser is not intended as a 'softcode' programming language. It does not + have things like loops and conditionals, for example. While you could in principle extend it to + do very advanced things and allow builders a lot of power, all-out coding is something + Evennia expects you to do in a proper text editor, outside of the game, not from inside it. +``` -- `$random([minval, maxval])` - produce a random number. `$random()` will give a random - number between 0 and 1. Giving a min/maxval will give a random value between these numbers. - If only one number is given, a random value from 0...number will be given. - The result will be an int or a float depending on if you give decimals or not. -- `$pad(text[, width, align, fillchar])` - this will pad content. `$pad("Hello", 30, c, -)` - will lead to a text centered in a 30-wide block surrounded by `-` characters. -- `$crop(text, width=78, suffix='[...]')` - this will crop a text longer than the width, - ending it with a `[...]`-suffix that also fits within the width. -- `$space(num)` - this will insert `num` spaces. -- `$clr(startcolor, text[, endcolor])` - color text. The color is given with one or two characters - without the preceeding `|`. If no endcolor is given, the string will go back to neutral. - so `$clr(r, Hello)` is equivalent to `|rHello|n`. - -### Protfuncs - -Evennia applies the parser on the keys and values of [Prototypes definitions](./Prototypes). This -is mainly used for in-game protoype building. The prototype keys/values are parsed with the -`FuncParser.parser_to_any` method so the user can set non-strings on prototype keys. - -See the prototype documentation for which protfuncs are available. - ## Using the FuncParser You can apply inline function parsing to any string. The @@ -124,22 +82,37 @@ parsed_string = parser.parser(input_string, raise_errors=False, parser = FuncParser(["game.myfuncparser_callables", "game.more_funcparser_callables"]) ``` -- `callables` - This is either a `dict` mapping `{"funcname": callable, ...}`, a python path to -a module or a list of such paths. If one or more paths, all top-level callables (whose name -does not start with an underscore) in that module are used to build the mapping automatically. +Here, `callables` points to a collection of normal Python functions (see next section) for you to make +available to the parser as you parse strings with it. It can either be +- A `dict` of `{"functionname": callable, ...}`. This allows you do pick and choose exactly which callables + to include and how they should be named. Do you want a callable to be available under more than one name? + Just add it multiple times to the dict, with a different key. +- A `module` or (more commonly) a `python-path` to a module. This module can define a dict + `FUNCPARSER_CALLABLES = {"funcname": callable, ...}` - this will be imported and used like ther `dict` above. + If no such variable is defined, _every_ top-level function in the module (whose name doesn't start with + an underscore `_`) will be considered a suitable callable. The name of the function will be the `$funcname` + by which it can be called. +- A `list` of modules/paths. This allows you to pull in modules from many sources for your parsing. + +The other arguments to the parser: + - `raise_errors` - By default, any errors from a callable will be quietly ignored and the result - will be that the failing function call will show as if it was escaped. If `raise_errors` is set, - then parsing will stop and the error raised. It'd be up to you to handle this properly. -- `escape` - Returns a string where every `$func(...)` has been escaped as `\$func()`. This makes the - string safe from further parsing. + will be that the failing function call will show verbatim. If `raise_errors` is set, + then parsing will stop and whatever exception happened will be raised. It'd be up to you to handle + this properly. +- `escape` - Returns a string where every `$func(...)` has been escaped as `\$func()`. - `strip` - Remove all `$func(...)` calls from string (as if each returned `''`). - `return_str` - When `True` (default), `parser` always returns a string. If `False`, it may return the return value of a single function call in the string. This is the same as using the `.parse_to_any` method. - The `**default/reserved_keywords` are optional and allow you to pass custom data into _every_ function call. This is great for including things like the current session or config options. Defaults can be - replaced if the user gives the same-named kwarg in the string's function call. Reserved kwargs - are always passed, ignoring defaults or what the user passed. + replaced if the user gives the same-named kwarg in the string's function call. Reserved kwargs are always passed, + ignoring defaults or what the user passed. In addition, the `funcparser` and `raise_errors` + reserved kwargs are always passed - the first is a back-reference to the `FuncParser` instance and the second + is the `raise_errors` boolean passed into `FuncParser.parse`. + +Here's an example of using the default/reserved keywords: ```python @@ -151,12 +124,16 @@ parser = funcparser.FuncParser({"test": _test}, mydefault=2) result = parser.parse("$test(foo, bar=4)", myreserved=[1, 2, 3]) ``` +Here the callable will be called as -Here the callable will be called as `_test('foo', bar='4', mydefault=2, myreserved=[1, 2, 3])`. -Note that everything given in the `$test(...)` call will enter the callable as strings. The -kwargs passed outside will be passed as whatever type they were given as. The `mydef` kwarg -could be overwritten by `$test(mydefault=...)` but `myreserved` will always be sent as-is, ignoring -any same-named kwarg given to `$test`. +```python + _test('foo', bar='4', mydefault=2, myreserved=[1, 2, 3], + funcparser=, raise_errrors=False) +``` + +The `mydefault=2` kwarg could be overwritten if we made the call as `$test(mydefault=...)` +but `myreserved=[1, 2, 3]` will _always_ be sent as-is and will override a call `$test(myreserved=...)`. +The `funcparser`/`raise_errors` kwargs are also always included as reserved kwargs. ## Defining custom callables @@ -168,134 +145,244 @@ def funcname(*args, **kwargs): return something ``` -As said, the input from the top-level string call will always be a string. However, if you -nest functions the input may be the return from _another_ callable. This may not be a string. -Since you should expect users to mix and match function calls, you must make sure your callables -gracefully can handle any input type. +> The `*args` and `**kwargs` must always be included. If you are unsure how `*args` and `**kwargs` work in Python, +> [read about them here](https://www.digitalocean.com/community/tutorials/how-to-use-args-and-kwargs-in-python-3). -On error, return an empty/default value or raise `evennia.utils.funcparser.ParsingError` to completely -stop the parsing at any nesting depth (the `raise_errors` boolean will determine what happens). +The input from the innermost `$funcname(...)` call in your callable will always be a `str`. Here's +an example of an `$toint` function; it converts numbers to integers. -Any type can be returned from the callable, but if its embedded in a longer string (or parsed without -`return_str=True`), the final outcome will always be a string. + "There's a $toint(22.0)% chance of survival." -First, here are two useful tools for converting strings to other Python types in a safe way: +What will enter the `$toint` callable (as `args[0]`) is the _string_ `"22.0"`. The function is responsible +for converting this to a float so it can operate on it. And also to properly handle invalid inputs (like +non-numbers). Common is to just return the input as-is or return the empty string. + +If you want to mark an error, raise `evennia.utils.funcparser.ParsingError`. This stops the entire parsing +of the string and may or may not raise the exception depending on what you set `raise_errors` to when you +created the parser. + +However, if you _nest_ functions, the return of the innermost function may be something other than +a string. Let's introduce the `$eval` function, which evaluates simple expressions using +Python's `literal_eval` and/or `simple_eval`. + + "There's a $toint($eval(10 * 2.2))% chance of survival." + +Since the `$eval` is the innermost call, it will get a sring as input - the string `"10 * 2.2"`. +It evaluates this and returns the `float` `22.0`. This time the outermost `$toint` will be called with +this `float` instead of with a string. + +> Since you don't know in which order users will nest or not nest your callables, it's important to +> safely validate your inputs. See the next section for useful tools to do this. + +In these examples, the result will be embedded in the larger string, so the result of the entire parsing +will be a string: + +```python + parser.parse(above_string) + "There's a 22% chance of survival." +``` + +However, if you use the `parse_to_any` (or `parse(..., return_str=True)`) and _don't add any extra string around the outermost function call_, +you'll get the return type of the outermost callable back: + +```python +parser.parse_to_any("$toint($eval(10 * 2.2)%") +"22%" +parser.parse_to_any("$toint($eval(10 * 2.2)") +22 +``` + +### Safe convertion of inputs + +Since you don't know in which order users may use your callables, they should always check the types +of its inputs and convert it to type the callable needs. Note also that this limits what inputs you can +support since some things (such as complex classes/callables etc) are just not safe/possible to +convert from string representation. + +In `evennia.utils.utils` is a helper called +[safe_convert_to_types](api.evennia.utils.utils#evennia.utils.utils.safe_convert_to_types). This function +automates the conversion of simple data types in a safe way: + +```python + +from evennia.utils.utils import safe_convert_to_types + +def _process_callable(*args, **kwargs): + """ + A callable with a lot of custom options + + $process(expression, local, extra=34, extra2=foo) + + """ + args, kwargs = safe_convert_to_type( + (('py', 'py'), {'extra1': int, 'extra2': str}), + *args, **kwargs) + + # args/kwargs should be correct types now + +``` + +In other words, + +```python + +args, kwargs = safe_convert_to_type( + (tuple_of_arg_converters, dict_of_kwarg_converters), *args, **kwargs) +``` + +Each converter should be a callable taking one argument - this will be the arg/kwarg-value to convert. The +special converter `"py"` will try to convert a string argument to a Python structure with the help of the +following tools (which you may also find useful to experiment with on your own): - [ast.literal_eval](https://docs.python.org/3.8/library/ast.html#ast.literal_eval) is an in-built Python function. It _only_ supports strings, bytes, numbers, tuples, lists, dicts, sets, booleans and `None`. That's it - no arithmetic or modifications of data is allowed. This is good for converting individual values and lists/dicts from the input line to real Python objects. -- [simpleeval](https://pypi.org/project/simpleeval/) is imported by Evennia. This allows for safe evaluation - of simple (and thus safe) expressions. One can operate on numbers and strings with +-/* as well - as do simple comparisons like `4 > 3` and more. It does _not_ accept more complex containers like - lists/dicts etc, so the two are complementary to each other. - -First we try `literal_eval`. This also illustrates how input types work. - -```python -from ast import literal_eval - -def _literal(*args, **kwargs): - if args: - try: - return literal_eval(args[0]) - except ValueError: - pass - return '' - -def _add(*args, **kwargs): - if len(args) > 1: - return args[0] + args[1] - return '' - -parser = FuncParser({"literal": _literal, "add": _add}) -``` - -We first try to add two numbers together straight up - -```python -parser.parse("$add(5, 10)") -"510" -``` -The result is that we concatenated the strings "5" + "10" which is not what we wanted. This -because the arguments from the top level string always enter the callable as strings. We next -try to convert each input value: - -```python -parser.parse("$add($lit(5), $lit(10))") -"15" -parser.parse_to_any("$add($lit(5), $lit(10))") -15 -parser.parse_to_any("$add($lit(5), $lit(10)) and extra text") -"15 and extra text" -``` -Now we correctly convert the strings to numbers and add them together. The result is still a string unless -we use `parse_to_any` (or `.parse(..., return_str=False)`). If we include the call as part of a bigger string, -the outcome is always be a string. - -In this case, `simple_eval` makes things easier: - -```python -from simpleeval import simple_eval - -def _eval((*args, **kwargs): - if args: - try: - return simple_eval(args[0]) - except Exception as err: - return f"" - -parser = FuncParser({"eval": _eval}) -parser.parse_to_any("5 + 10") -10 - -``` - -This is a lot more natural in this case, but `literal_eval` can convert things like lists/dicts that the -`simple_eval` cannot. Here we also tried out a different way to handle errors - by letting an error replace -the `$func`-call directly in the string. This is not always suitable. +- [simpleeval](https://pypi.org/project/simpleeval/) is a third-party tool included with Evennia. This + allows for evaluation of simple (and thus safe) expressions. One can operate on numbers and strings + with +-/* as well as do simple comparisons like `4 > 3` and more. It does _not_ accept more complex + containers like lists/dicts etc, so this and `literal_eval` are complementary to each other. ```warning:: - It may be tempting to run Python's in-built `eval()` or `exec()` commands on the input in order to convert - it from a string to regular Python objects. NEVER DO THIS. The parser is intended for untrusted users (if - you were trusted you'd have access to Python already). Letting untrusted users pass strings to eval/exec - is a MAJOR security risk. It allows the caller to effectively run arbitrary Python code on your server. - This is the way to maliciously deleted hard drives. Just don't do it and sleep better at night. + It may be tempting to run use Python's in-built `eval()` or `exec()` functions as converters since these + are able to convert any valid Python source code to Python. NEVER DO THIS unless you really, really + know ONLY developers will ever send input to the callable. The parser is intended + for untrusted users (if you were trusted you'd have access to Python already). Letting untrusted users + pass strings to eval/exec is a MAJOR security risk. It allows the caller to effectively run arbitrary + Python code on your server. This is the way to maliciously deleted hard drives. Just don't do it and + sleep better at night. ``` -## Example: +## Default callables -An +These are some example callables you can import and add your parser. They are divided into +global-level dicts in `evennia.utils.funcparser`. Just import the dict(s) and merge/add one or +more to them when you create your `FuncParser` instance to have those callables be available. + +### `evennia.utils.funcparser.FUNCPARSER_CALLABLES` + +These are the 'base' callables. + +- `$eval(expression)` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_eval) - + this uses `literal_eval` and `simple_eval` (see previous section) attemt to convert a string expression + to a python object. This handles e.g. lists of literals `[1, 2, 3]` and simple expressions like `"1 + 2"`. +- `$toint(number)` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_toint)) - + always converts an output to an integer, if possible. +- `$add/sub/mult/div(obj1, obj2)` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_add))) - + this adds/subtracts/multiplies and divides to elements together. While simple addition could be done with + `$eval`, this could for example be used also to add two lists together, which is not possible with `eval`; + for example `$add($eval([1,2,3]), $eval([4,5,6])) -> [1, 2, 3, 4, 5, 6]`. +- `$round(float, significant)` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_round) - + rounds an input float into the number of provided significant digits. For example `$round(3.54343, 3) -> 3.543`. +- `$random([start, [end]])` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_random)) - + this works like the Python `random()` function, but will randomize to an integer value if both start/end are + integers. Without argument, will return a float between 0 and 1. +- `$randint([start, [end]])` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_randint)) - + works like the `randint()` python function and always returns an integer. +- `$choice(list)` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_choice)) - + the input will automatically be parsed the same way as `$eval` and is expected to be an iterable. A random + element of this list will be returned. +- `$pad(text[, width, align, fillchar])` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_pad)) - + this will pad content. `$pad("Hello", 30, c, -)` will lead to a text centered in a 30-wide block surrounded by `-` + characters. +- `$crop(text, width=78, suffix='[...]')` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_crop)) - + this will crop a text longer than the width, by default ending it with a `[...]`-suffix that also fits within + the width. If no width is given, the client width or `settings.DEFAULT_CLIENT_WIDTH` will be used. +- `$space(num)` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_space)) - + this will insert `num` spaces. +- `$just(string, width=40, align=c, indent=2)` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_justify)) - + justifies the text to a given width, aligning it left/right/center or 'f' for full (spread text across width). +- `$ljust` - shortcut to justify-left. Takes all other kwarg of `$just`. +- `$rjust` - shortcut to right justify. +- `$cjust` - shortcut to center justify. +- `$clr(startcolor, text[, endcolor])`([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_clr)) - + color text. The color is given with one or two characters without the preceeding `|`. If no endcolor is + given, the string will go back to neutral, so `$clr(r, Hello)` is equivalent to `|rHello|n`. + +### `evennia.utils.funcparser.SEARCHING_CALLABLES` + +These are callables that requires access-checks in order to search for objects. So they require some +extra reserved kwargs to be passed when running the parser: + +```python +parser.parse_to_any(string, caller=, access="control", ...)` + +``` +The `caller` is required, it's the the object to do the access-check for. The `access` kwarg is the + [lock type](Locks) to check, default being `"control"`. + +- `$search(query,type=account|script,return_list=False)` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_search)) - + this will look up and try to match an object by key or alias. Use the `type` kwarg to + search for `account` or `script` instead. By default this will return nothing if there are more than one + match; if `return_list` is `True` a list of 0, 1 or more matches will be returned instead. +- `$obj(query)`, `$dbref(query)` - legacy aliases for `$search`. +- `$objlist(query)` - legacy alias for `$search`, always returning a list. + + +### `evennia.utils.funcparser.ACTOR_STANCE_CALLABLES` + +These are used to implement actor-stance emoting. They are used by the +[DefaultObject.msg_contents](api:evennia.objects.objects#evennia.objects.objects.DefaultObject.msg_contents) method +by default. + +These all require extra kwargs be passed into the parser: + +```python +parser.parse(string, caller=, receiver=, mapping={'key': , ...}) +``` + +Here the `caller` is the one sending the message and `receiver` the one to see it. The `mapping` contains +references to other objects accessible via these callables. + +- `$you([key])` ([code](api:evennia.utils.funcparser#evennia.utils.funcparser.funcparser_callable_you)) - + if no `key` is given, this represents the `caller`, otherwise an object from `mapping` + will be used. As this message is sent to different recipients, the `receiver` will change and this will + be replaced either with the string `you` (if you and the receiver is the same entity) or with the + result of `you_obj.get_display_name(looker=receiver)`. This allows for a single string to echo differently + depending on who sees it, and also to reference other people in the same way. +- `$You([key])` - same as `$you` but always capitalized. +- `$conj(verb)` - conjugates a verb between 2nd person presens to 3rd person presence depending on who + sees the string. For example `$You() $conj(smiles).` will show as "You smile." and "Tom smiles." depending + on who sees it. This makes use of the tools in [evennia.utils.verb_conjugation](api.evennia.utils.verb_conjugation) + to do this, and only works for English verbs. + +### Example + +Here's an example of including the default callables together with two custom ones. ```python from evennia.utils import funcparser from evennia.utils import gametime -def _header(*args, **kwargs): +def _dashline(*args, **kwargs): if args: - return "\n-------- {args[0]} --------" + return f"\n-------- {args[0]} --------" return '' def _uptime(*args, **kwargs): return gametime.uptime() callables = { - "header": _header, - "uptime": _uptime + "dashline": _dashline, + "uptime": _uptime, + **funcparser.FUNCPARSER_CALLABLES, + **funcparser.ACTOR_STANCE_CALLABLES, + **funcparser.SEARCHING_CALLABLES } parser = funcparser.FuncParser(callables) -string = "This is the current uptime:$header($uptime() seconds)" +string = "This is the current uptime:$dashline($toint($uptime()) seconds)" result = parser.parse(string) ``` -Above we define two callables `_header` and `_uptime` and map them to names `"header"` and `"uptime"`, -which is what we then can call as `$header` and `$uptime` in the string. +Above we define two callables `_dashline` and `_uptime` and map them to names `"dashline"` and `"uptime"`, +which is what we then can call as `$header` and `$uptime` in the string. We also have access to +all the defaults (like `$toint()`). -We nest the functions so the parsed result of the above would be something like this: +The parsed result of the above would be something like this: ``` This is the current uptime: diff --git a/evennia/objects/objects.py b/evennia/objects/objects.py index b1482a3dda..cbd21ed79b 100644 --- a/evennia/objects/objects.py +++ b/evennia/objects/objects.py @@ -803,7 +803,7 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase): # actor-stance replacements inmessage = _MSG_CONTENTS_PARSER.parse( inmessage, raise_errors=True, return_string=True, - you=you, receiver=receiver, mapping=mapping) + caller=you, receiver=receiver, mapping=mapping) # director-stance replacements outmessage = inmessage.format( diff --git a/evennia/utils/funcparser.py b/evennia/utils/funcparser.py index e19bdf7f75..e281467d09 100644 --- a/evennia/utils/funcparser.py +++ b/evennia/utils/funcparser.py @@ -46,11 +46,10 @@ import inspect import random from functools import partial from django.conf import settings -from ast import literal_eval -from simpleeval import simple_eval from evennia.utils import logger from evennia.utils.utils import ( - make_iter, callables_from_module, variable_from_module, pad, crop, justify) + make_iter, callables_from_module, variable_from_module, pad, crop, justify, + safe_convert_to_types) from evennia.utils import search from evennia.utils.verb_conjugation.conjugate import verb_actor_stance_components @@ -233,11 +232,15 @@ class FuncParser: f"(available: {available})") return str(parsedfunc) + nargs = len(args) + # build kwargs in the proper priority order - kwargs = {**self.default_kwargs, **kwargs, **reserved_kwargs} + kwargs = {**self.default_kwargs, **kwargs, **reserved_kwargs, + **{'funcparser': self, "raise_errors": raise_errors}} try: - return func(*args, **kwargs) + ret = func(*args, **kwargs) + return ret except ParsingError: if raise_errors: raise @@ -601,19 +604,8 @@ def funcparser_callable_eval(*args, **kwargs): `$py(3 + 4)` """ - if not args: - return '' - inp = args[0] - if not isinstance(inp, str): - # already converted - return inp - try: - return literal_eval(inp) - except Exception: - try: - return simple_eval(inp) - except Exception: - return inp + args, kwargs = safe_convert_to_types(("py", {}) , *args, **kwargs) + return args[0] if args else '' def funcparser_callable_toint(*args, **kwargs): @@ -640,28 +632,23 @@ def _apply_operation_two_elements(*args, operator="+", **kwargs): better for non-list arithmetic. """ + args, kwargs = safe_convert_to_types((('py', 'py'), {}), *args, **kwargs) if not len(args) > 1: return '' val1, val2 = args[0], args[1] - # try to convert to python structures, otherwise, keep as strings - if isinstance(val1, str): - try: - val1 = literal_eval(val1.strip()) - except Exception: - pass - if isinstance(val2, str): - try: - val2 = literal_eval(val2.strip()) - except Exception: - pass - if operator == "+": - return val1 + val2 - elif operator == "-": - return val1 - val2 - elif operator == "*": - return val1 * val2 - elif operator == "/": - return val1 / val2 + try: + if operator == "+": + return val1 + val2 + elif operator == "-": + return val1 - val2 + elif operator == "*": + return val1 * val2 + elif operator == "/": + return val1 / val2 + except Exception: + if kwargs.get('raise_errors'): + raise + return '' def funcparser_callable_add(*args, **kwargs): @@ -705,21 +692,15 @@ def funcparser_callable_round(*args, **kwargs): """ if not args: return '' - inp, *significant = args - significant = significant[0] if significant else '0' - lit_inp = inp - if isinstance(inp, str): - try: - lit_inp = literal_eval(inp) - except Exception: - return inp + args, _ = safe_convert_to_types(((float, int), {}) *args, **kwargs) + + num, *significant = args + significant = significant[0] if significant else 0 try: - int(significant) - except Exception: - significant = 0 - try: - round(lit_inp, significant) + round(num, significant) except Exception: + if kwargs.get('raise_errors'): + raise return '' def funcparser_callable_random(*args, **kwargs): @@ -744,35 +725,32 @@ def funcparser_callable_random(*args, **kwargs): - `$random(5, 10.0)` - random value [5..10] (float) """ + args, _ = safe_convert_to_types((('py', 'py'), {}), *args, **kwargs) + nargs = len(args) if nargs == 1: # only maxval given - minval, maxval = "0", args[0] + minval, maxval = 0, args[0] elif nargs > 1: minval, maxval = args[:2] else: - minval, maxval = ("0", "1") + minval, maxval = 0, 1 - if "." in minval or "." in maxval: - # float mode - try: - minval, maxval = float(minval), float(maxval) - except ValueError: - minval, maxval = 0, 1 - return minval + maxval * random.random() - else: - # int mode - try: - minval, maxval = int(minval), int(maxval) - except ValueError: - minval, maxval = 0, 1 - return random.randint(minval, maxval) + try: + if isinstance(minval, float) or isinstance(maxval, float): + return minval + maxval * random.random() + else: + return random.randint(minval, maxval) + except Exception: + if kwargs.get('raise_errors'): + raise + return '' def funcparser_callable_randint(*args, **kwargs): """ Usage: $randint(start, end): - Legacy alias - alwas returns integers. + Legacy alias - always returns integers. """ return int(funcparser_callable_random(*args, **kwargs)) @@ -796,10 +774,13 @@ def funcparser_callable_choice(*args, **kwargs): """ if not args: return '' - inp = args[0] - if not isinstance(inp, str): - inp = literal_eval(inp) - return random.choice(inp) + args, _ = safe_convert_to_types(('py', {}), *args, **kwargs) + try: + return random.choice(args[0]) + except Exception: + if kwargs.get('raise_errors'): + raise + return '' def funcparser_callable_pad(*args, **kwargs): @@ -819,6 +800,9 @@ def funcparser_callable_pad(*args, **kwargs): """ if not args: return '' + args, kwargs = safe_convert_to_types( + ((str, int, str, str), {'width': int, 'align': str, 'fillchar': str}), *args, **kwargs) + text, *rest = args nrest = len(rest) try: @@ -833,22 +817,6 @@ def funcparser_callable_pad(*args, **kwargs): return pad(str(text), width=width, align=align, fillchar=fillchar) -def funcparser_callable_space(*args, **kwarg): - """ - Usage: $space(43) - - Insert a length of space. - - """ - if not args: - return '' - try: - width = int(args[0]) - except TypeError: - width = 1 - return " " * width - - def funcparser_callable_crop(*args, **kwargs): """ FuncParser callable. Crops ingoing text to given widths. @@ -877,6 +845,22 @@ def funcparser_callable_crop(*args, **kwargs): return crop(str(text), width=width, suffix=str(suffix)) +def funcparser_callable_space(*args, **kwarg): + """ + Usage: $space(43) + + Insert a length of space. + + """ + if not args: + return '' + try: + width = int(args[0]) + except TypeError: + width = 1 + return " " * width + + def funcparser_callable_justify(*args, **kwargs): """ Justify text across a width, default across screen width. @@ -948,6 +932,7 @@ def funcparser_callable_clr(*args, **kwargs): """ if not args: return '' + startclr, text, endclr = '', '', '' if len(args) > 1: # $clr(pre, text, post)) @@ -1045,7 +1030,7 @@ def funcparser_callable_search_list(*args, caller=None, access="control", **kwar return_list=True, **kwargs) -def funcparser_callable_you(*args, you=None, receiver=None, mapping=None, capitalize=False, **kwargs): +def funcparser_callable_you(*args, caller=None, receiver=None, mapping=None, capitalize=False, **kwargs): """ Usage: $you() or $you(key) @@ -1053,19 +1038,19 @@ def funcparser_callable_you(*args, you=None, receiver=None, mapping=None, capita of the caller for others. Kwargs: - you (Object): The 'you' in the string. This is used unless another + caller (Object): The 'you' in the string. This is used unless another you-key is passed to the callable in combination with `mapping`. receiver (Object): The recipient of the string. mapping (dict, optional): This is a mapping `{key:Object, ...}` and is used to find which object `$you(key)` refers to. If not given, the - `you` kwarg is used. + `caller` kwarg is used. capitalize (bool): Passed by the You helper, to capitalize you. Returns: str: The parsed string. Raises: - ParsingError: If `you` and `receiver` were not supplied. + ParsingError: If `caller` and `receiver` were not supplied. Notes: The kwargs should be passed the to parser directly. @@ -1076,7 +1061,7 @@ def funcparser_callable_you(*args, you=None, receiver=None, mapping=None, capita - `With a grin, $you() $conj(jump) at $you(tommy).` - The You-object will see "With a grin, you jump at Tommy." + The caller-object will see "With a grin, you jump at Tommy." Tommy will see "With a grin, CharName jumps at you." Others will see "With a grin, CharName jumps at Tommy." @@ -1084,17 +1069,17 @@ def funcparser_callable_you(*args, you=None, receiver=None, mapping=None, capita if args and mapping: # this would mean a $you(key) form try: - you = mapping.get(args[0]) + caller = mapping.get(args[0]) except KeyError: pass - if not (you and receiver): - raise ParsingError("No you-object or receiver supplied to $you callable.") + if not (caller and receiver): + raise ParsingError("No caller or receiver supplied to $you callable.") capitalize = bool(capitalize) - if you == receiver: + if caller == receiver: return "You" if capitalize else "you" - return you.get_display_name(looker=receiver) if hasattr(you, "get_display_name") else str(you) + return caller.get_display_name(looker=receiver) if hasattr(caller, "get_display_name") else str(caller) def funcparser_callable_You(*args, you=None, receiver=None, mapping=None, capitalize=True, **kwargs): @@ -1106,14 +1091,14 @@ def funcparser_callable_You(*args, you=None, receiver=None, mapping=None, capita *args, you=you, receiver=receiver, mapping=mapping, capitalize=capitalize, **kwargs) -def funcparser_callable_conjugate(*args, you=None, receiver=None, **kwargs): +def funcparser_callable_conjugate(*args, caller=None, receiver=None, **kwargs): """ $conj(verb) Conjugate a verb according to if it should be 2nd or third person. Kwargs: - you_obj (Object): The object who represents 'you' in the string. - you_target (Object): The recipient of the string. + caller (Object): The object who represents 'you' in the string. + receiver (Object): The recipient of the string. Returns: str: The parsed string. @@ -1139,11 +1124,11 @@ def funcparser_callable_conjugate(*args, you=None, receiver=None, **kwargs): """ if not args: return '' - if not (you and receiver): - raise ParsingError("No youj/receiver supplied to $conj callable") + if not (caller and receiver): + raise ParsingError("No caller/receiver supplied to $conj callable") second_person_str, third_person_str = verb_actor_stance_components(args[0]) - return second_person_str if you == receiver else third_person_str + return second_person_str if caller == receiver else third_person_str # these are made available as callables by adding 'evennia.utils.funcparser' as diff --git a/evennia/utils/tests/test_funcparser.py b/evennia/utils/tests/test_funcparser.py index 315c70bc7c..bde79b24b8 100644 --- a/evennia/utils/tests/test_funcparser.py +++ b/evennia/utils/tests/test_funcparser.py @@ -14,6 +14,8 @@ from evennia.utils import funcparser, test_resources def _test_callable(*args, **kwargs): + kwargs.pop('funcparser', None) + kwargs.pop('raise_errors', None) argstr = ", ".join(args) kwargstr = "" if kwargs: @@ -311,10 +313,10 @@ class TestDefaultCallables(TestCase): """ mapping = {"char1": self.obj1, "char2": self.obj2} - ret = self.parser.parse(string, you=self.obj1, receiver=self.obj1, mapping=mapping, + ret = self.parser.parse(string, caller=self.obj1, receiver=self.obj1, mapping=mapping, raise_errors=True) self.assertEqual(expected_you, ret) - ret = self.parser.parse(string, you=self.obj1, receiver=self.obj2, mapping=mapping, + ret = self.parser.parse(string, caller=self.obj1, receiver=self.obj2, mapping=mapping, raise_errors=True) self.assertEqual(expected_them, ret) @@ -346,10 +348,26 @@ class TestDefaultCallables(TestCase): def test_random(self): string = "$random(1,10)" - ret = self.parser.parse(string, raise_errors=True) - ret = int(ret) + ret = self.parser.parse_to_any(string, raise_errors=True) self.assertTrue(1 <= ret <= 10) + string = "$random()" + ret = self.parser.parse_to_any(string, raise_errors=True) + self.assertTrue(0 <= ret <= 1) + + string = "$random(1.0, 3.0)" + for i in range(1000): + ret = self.parser.parse_to_any(string, raise_errors=True) + self.assertTrue(isinstance(ret, float)) + print("ret:", ret) + self.assertTrue(1.0 <= ret <= 3.0) + + def test_randint(self): + string = "$randint(1.0, 3.0)" + ret = self.parser.parse_to_any(string, raise_errors=True) + self.assertTrue(isinstance(ret, int)) + self.assertTrue(1.0 <= ret <= 3.0) + def test_nofunc(self): self.assertEqual( self.parser.parse("as$382ewrw w we w werw,|44943}"), diff --git a/evennia/utils/tests/test_utils.py b/evennia/utils/tests/test_utils.py index b74668845d..98fa362485 100644 --- a/evennia/utils/tests/test_utils.py +++ b/evennia/utils/tests/test_utils.py @@ -8,6 +8,7 @@ TODO: Not nearly all utilities are covered yet. import os.path import random +from parameterized import parameterized import mock from django.test import TestCase from datetime import datetime @@ -385,3 +386,43 @@ class TestPercent(TestCase): self.assertEqual(utils.percent(3, 1, 1), "0.0%") self.assertEqual(utils.percent(3, 0, 1), "100.0%") self.assertEqual(utils.percent(-3, 0, 1), "0.0%") + + +class TestSafeConvert(TestCase): + """ + Test evennia.utils.utils.safe_convert_to_types + + """ + + @parameterized.expand([ + (('1', '2', 3, 4, '5'), {'a': 1, 'b': '2', 'c': 3}, + ((int, float, str, int), {'a': int, 'b': float}), # " + (1, 2.0, '3', 4, '5'), {'a': 1, 'b': 2.0, 'c': 3}), + (('1 + 2', '[1, 2, 3]', [3, 4, 5]), {'a': '3 + 4', 'b': 5}, + (('py', 'py', 'py'), {'a': 'py', 'b': 'py'}), + (3, [1, 2, 3], [3, 4, 5]), {'a': 7, 'b': 5}), + ]) + def test_conversion(self, args, kwargs, converters, expected_args, expected_kwargs): + """ + Test the converter with different inputs + + """ + result_args, result_kwargs = utils.safe_convert_to_types( + converters, *args, raise_errors=True, **kwargs) + self.assertEqual(expected_args, result_args) + self.assertEqual(expected_kwargs, result_kwargs) + + def test_conversion__fail(self): + """ + Test failing conversion + + """ + from evennia.utils.funcparser import ParsingError + + with self.assertRaises(ValueError): + utils.safe_convert_to_types( + (int, ), *('foo', ), raise_errors=True) + + with self.assertRaises(ParsingError) as err: + utils.safe_convert_to_types( + ('py', {}), *('foo', ), raise_errors=True) diff --git a/evennia/utils/utils.py b/evennia/utils/utils.py index 1ee0a5af06..c4cc1cccf9 100644 --- a/evennia/utils/utils.py +++ b/evennia/utils/utils.py @@ -20,6 +20,8 @@ import traceback import importlib import importlib.util import importlib.machinery +from ast import literal_eval +from simpleeval import simple_eval from unicodedata import east_asian_width from twisted.internet.task import deferLater from twisted.internet.defer import returnValue # noqa - used as import target @@ -2390,3 +2392,104 @@ def interactive(func): return ret return decorator + + +def safe_convert_to_types(converters, *args, raise_errors=True, **kwargs): + """ + Helper function to safely convert inputs to expected data types. + + Args: + converters (tuple): A tuple `((converter, converter,...), {kwarg: converter, ...})` to + match a converter to each element in `*args` and `**kwargs`. + Each converter will will be called with the arg/kwarg-value as the only argument. + If there are too few converters given, the others will simply not be converter. If the + converter is given as the string 'py', it attempts to run + `safe_eval`/`literal_eval` on the input arg or kwarg value. It's possible to + skip the arg/kwarg part of the tuple, an empty tuple/dict will then be assumed. + *args: The arguments to convert with `argtypes`. + raise_errors (bool, optional): If set, raise any errors. This will + abort the conversion at that arg/kwarg. Otherwise, just skip the + conversion of the failing arg/kwarg. This will be set by the FuncParser if + this is used as a part of a FuncParser callable. + **kwargs: The kwargs to convert with `kwargtypes` + + Returns: + tuple: `(args, kwargs)` in converted form. + + Raises: + utils.funcparser.ParsingError: If parsing failed in the `'py'` + converter. This also makes this compatible with the FuncParser + interface. + any: Any other exception raised from other converters, if raise_errors is True. + + Notes: + This function is often used to validate/convert input from untrusted sources. For + security, the "py"-converter is deliberately limited and uses `safe_eval`/`literal_eval` + which only supports simple expressions or simple containers with literals. NEVER + use the python `eval` or `exec` methods as a converter for any untrusted input! Allowing + untrusted sources to execute arbitrary python on your server is a severe security risk, + + Example: + :: + + $funcname(1, 2, 3.0, c=[1,2,3]) + + def _funcname(*args, **kwargs): + args, kwargs = safe_convert_input(((int, int, float), {'c': 'py'}), *args, **kwargs) + # ... + + """ + def _safe_eval(inp): + if not inp: + return '' + if not isinstance(inp, str): + # already converted + return inp + + try: + return literal_eval(inp) + except Exception as err: + literal_err = f"{err.__class__.__name__}: {err}" + try: + return simple_eval(inp) + except Exception as err: + simple_err = f"{str(err.__class__.__name__)}: {err}" + pass + + if raise_errors: + from evennia.utils.funcparser import ParsingError + err = (f"Errors converting '{inp}' to python:\n" + f"literal_eval raised {literal_err}\n" + f"simple_eval raised {simple_err}") + raise ParsingError(err) + + # handle an incomplete/mixed set of input converters + if not converters: + return args, kwargs + arg_converters, *kwarg_converters = converters + arg_converters = make_iter(arg_converters) + kwarg_converters = kwarg_converters[0] if kwarg_converters else {} + + # apply the converters + if args and arg_converters: + args = list(args) + arg_converters = make_iter(arg_converters) + for iarg, arg in enumerate(args[:len(arg_converters)]): + converter = arg_converters[iarg] + converter = _safe_eval if converter in ('py', 'python') else converter + try: + args[iarg] = converter(arg) + except Exception: + if raise_errors: + raise + args = tuple(args) + if kwarg_converters and isinstance(kwarg_converters, dict): + for key, converter in kwarg_converters.items(): + converter = _safe_eval if converter in ('py', 'python') else converter + if key in {**kwargs}: + try: + kwargs[key] = converter(kwargs[key]) + except Exception: + if raise_errors: + raise + return args, kwargs