From 4cb6a140982a61710cc4ec10ac0aaa619bfd9a79 Mon Sep 17 00:00:00 2001 From: Petter Strandmark Date: Sat, 27 Jul 2019 18:49:54 +0200 Subject: [PATCH 1/4] Work on module. --- module.c | 76 +++++++++++++++++++++++++++++++++++++++++++------ test_quickjs.py | 41 +++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/module.c b/module.c index c6a09cc..2522dd0 100644 --- a/module.c +++ b/module.c @@ -10,6 +10,7 @@ typedef struct { JSContext *context; int has_time_limit; clock_t time_limit; + PyObject* module_loader; } ContextData; // The data of the type _quickjs.Object. @@ -33,6 +34,34 @@ typedef struct { clock_t limit; } InterruptData; +static JSModuleDef *js_module_loader(JSContext *ctx, + const char *module_name, void *opaque) { + ContextData* context = (ContextData*) opaque; + JSModuleDef* module = NULL; + if (context->module_loader == NULL || !PyCallable_Check(context->module_loader)) { + JS_ThrowReferenceError(ctx, "Could not load module \"%s\": no loader.", module_name); + return NULL; + } else { + PyObject* result = PyObject_CallFunction(context->module_loader, "s", module_name); + + if (result == NULL) { + JS_ThrowReferenceError(ctx, "Could not load module \"%s\": loader failed.", module_name); + } else if (!PyUnicode_Check(result)) { + JS_ThrowReferenceError(ctx, "Could not load module \"%s\": loader did not return a string.", module_name); + } else { + const char* code = PyUnicode_AsUTF8(result); + JSValue js_value = JS_Eval(ctx, code, strlen(code), module_name, JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); + if (!JS_IsException(js_value)) { + module = JS_VALUE_GET_PTR(js_value); + JS_FreeValue(ctx, js_value); + } + } + + Py_XDECREF(result); + } + return module; +} + // Returns nonzero if we should stop due to a time limit. static int js_interrupt_handler(JSRuntime *rt, void *opaque) { InterruptData *data = opaque; @@ -255,23 +284,26 @@ struct module_state {}; // Creates an instance of the _quickjs.Context class. static PyObject *context_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - ContextData *self; - self = (ContextData *)type->tp_alloc(type, 0); + ContextData *self = (ContextData *)type->tp_alloc(type, 0); if (self != NULL) { // We never have different contexts for the same runtime. This way, different // _quickjs.Context can be used concurrently. self->runtime = JS_NewRuntime(); - self->context = JS_NewContext(self->runtime); + self->context = JS_NewContext(self->runtime); self->has_time_limit = 0; self->time_limit = 0; + self->module_loader = NULL; + + JS_SetModuleLoaderFunc(self->runtime, NULL, js_module_loader, self); } return (PyObject *)self; } // Deallocates an instance of the _quickjs.Context class. -static void context_dealloc(ContextData *self) { +static void context_dealloc(ContextData *self) { JS_FreeContext(self->context); JS_FreeRuntime(self->runtime); + Py_XDECREF(self->module_loader); Py_TYPE(self)->tp_free((PyObject *)self); } @@ -284,15 +316,19 @@ static PyObject *context_eval_internal(ContextData *self, PyObject *args, int ev } // Perform the actual evaluation. We release the GIL in order to speed things up for certain - // use cases. If this module becomes more complicated and gains the capability to call Python - // function from JS, this needs to be reversed or improved. + // use cases. JSValue value; - Py_BEGIN_ALLOW_THREADS; InterruptData interrupt_data; setup_time_limit(self, &interrupt_data); - value = JS_Eval(self->context, code, strlen(code), "", eval_type); + if (self->module_loader != NULL && (eval_type & JS_EVAL_TYPE_MODULE)) { + // Can not release the GIL. We may have to call into Python code. + value = JS_Eval(self->context, code, strlen(code), "", eval_type); + } else { + Py_BEGIN_ALLOW_THREADS; + value = JS_Eval(self->context, code, strlen(code), "", eval_type); + Py_END_ALLOW_THREADS; + } teardown_time_limit(self); - Py_END_ALLOW_THREADS; return quickjs_to_python(self, value); } @@ -366,6 +402,24 @@ static PyObject *context_set_max_stack_size(ContextData *self, PyObject *args) { Py_RETURN_NONE; } +// _quickjs.Context.set_module_loade +// +// Sets the max stack size in bytes. +static PyObject *context_set_module_loader(ContextData *self, PyObject *args) { + PyObject* loader; + if (!PyArg_ParseTuple(args, "O", &loader)) { + return NULL; + } + if (!PyCallable_Check(loader)) { + PyErr_SetString(PyExc_TypeError, "Module loader must be callable."); + return NULL; + } + Py_XDECREF(self->module_loader); + self->module_loader = loader; + Py_INCREF(self->module_loader); + Py_RETURN_NONE; +} + // _quickjs.Context.memory // // Sets the CPU time limit of the context. This will be used in an interrupt handler. @@ -441,6 +495,10 @@ static PyMethodDef context_methods[] = { (PyCFunction)context_set_max_stack_size, METH_VARARGS, "Sets the maximum stack size in bytes. Default is 256kB."}, + {"set_module_loader", + (PyCFunction)context_set_module_loader, + METH_VARARGS, + "Sets the Python function that takes a module name and returns its source code."}, {"memory", (PyCFunction)context_memory, METH_NOARGS, "Returns the memory usage as a dict."}, {"gc", (PyCFunction)context_gc, METH_NOARGS, "Runs garbage collection."}, {NULL} /* Sentinel */ diff --git a/test_quickjs.py b/test_quickjs.py index 822c181..23c970b 100644 --- a/test_quickjs.py +++ b/test_quickjs.py @@ -54,13 +54,52 @@ def test_get(self): self.assertEqual(self.context.get("y"), "foo") self.assertEqual(self.context.get("z"), None) - def test_module(self): + def test_run_module(self): self.context.module(""" export function test() { return 42; } """) + def test_import_module_without_loader(self): + with self.assertRaisesRegex(quickjs.JSException, "no loader"): + self.context.module('import { foo } from "petter";') + + def test_set_module_loader_to_non_callable(self): + with self.assertRaises(TypeError): + self.context.set_module_loader(2) + + def test_module_does_not_return_code(self): + self.context.set_module_loader(lambda name: 1) + with self.assertRaisesRegex(quickjs.JSException, "did not return a string"): + self.context.module('import { foo } from "petter";') + + def _module_loader(self, name): + """Helper method that for module loading.""" + + if name == "petter": + return """ + export function foo() { + return 40; + } + """ + else: + raise ValueError("Unknown module.") + + def test_import_unknown_module(self): + self.context.set_module_loader(self._module_loader) + with self.assertRaisesRegex(quickjs.JSException, "loader failed"): + self.context.module('import { foo } from "somewhere_else";') + + def test_import_module(self): + self.context.set_module_loader(self._module_loader) + self.context.module(""" + import { foo } from "petter"; + (function test() { + return foo(); + })() + """) + def test_error(self): with self.assertRaisesRegex(quickjs.JSException, "ReferenceError: missing is not defined"): self.context.eval("missing + missing") From e1af2e92c41c7913417cb1a5be3321f349af2ce1 Mon Sep 17 00:00:00 2001 From: Petter Strandmark Date: Sun, 26 Jan 2020 20:20:07 +0100 Subject: [PATCH 2/4] Implement get() and dir() for Object. --- module.c | 32 ++++++++++++++++++++++++++++++++ test_quickjs.py | 23 ++++++++++++++++++----- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/module.c b/module.c index e0014bb..f4ad034 100644 --- a/module.c +++ b/module.c @@ -121,9 +121,41 @@ static PyObject *object_json(ObjectData *self) { return quickjs_to_python(self->context, json_string); } +// _quickjs.Object.get +// +// Retrieves a property from the JS object. +static PyObject *object_get(ObjectData *self, PyObject *args) { + const char *name; + if (!PyArg_ParseTuple(args, "s", &name)) { + return NULL; + } + JSValue value = JS_GetPropertyStr(self->context->context, self->object, name); + return quickjs_to_python(self->context, value); +} + +// _quickjs.Object.dir +// +// Returns a list of all properties of a JS object. +static PyObject *object_dir(ObjectData *self) { + JSPropertyEnum* props = 0; + uint32_t len; + JS_GetOwnPropertyNames(self->context->context, &props, &len, self->object, JS_GPN_STRING_MASK); + + PyObject* list = PyList_New(len); + for (int i=0; icontext->context, props[i].atom); + PyList_SetItem(list, i, Py_BuildValue("s", prop)); + JS_FreeCString(self->context->context, prop); + } + js_free(self->context->context, props); + return list; +} + // All methods of the _quickjs.Object class. static PyMethodDef object_methods[] = { {"json", (PyCFunction)object_json, METH_NOARGS, "Converts to a JSON string."}, + {"get", (PyCFunction)object_get, METH_VARARGS, "Retrieves a property."}, + {"dir", (PyCFunction)object_dir, METH_NOARGS, "Returns a list of all properties."}, {NULL} /* Sentinel */ }; diff --git a/test_quickjs.py b/test_quickjs.py index fe7ccf5..ca60a73 100644 --- a/test_quickjs.py +++ b/test_quickjs.py @@ -78,10 +78,10 @@ def test_module_does_not_return_code(self): def _module_loader(self, name): """Helper method that for module loading.""" - if name == "petter": + if name == "mymodule": return """ export function foo() { - return 40; + return 42; } """ else: @@ -95,11 +95,15 @@ def test_import_unknown_module(self): def test_import_module(self): self.context.set_module_loader(self._module_loader) self.context.module(""" - import { foo } from "petter"; - (function test() { + import { foo } from "mymodule"; + + function test() { return foo(); - })() + } + + globalThis.test = test; """) + self.assertEqual(self.context.get("test")(), 42) def test_error(self): with self.assertRaisesRegex(quickjs.JSException, "ReferenceError: missing is not defined"): @@ -277,6 +281,15 @@ def test_wrong_context(self): with self.assertRaisesRegex(ValueError, "Can not mix JS objects from different contexts."): f(d) + def test_dir(self): + d = self.context.eval("({a: 1, b: 2})") + self.assertEqual(d.dir(), ["a", "b"]) + + def test_get(self): + d = self.context.eval("({a: 1, b: 2})") + self.assertEqual(d.get("a"), 1) + self.assertEqual(d.get("b"), 2) + class FunctionTest(unittest.TestCase): def test_adder(self): From 8c68df9f82d9caf3f62e263158c1b080e7fdabaf Mon Sep 17 00:00:00 2001 From: Petter Strandmark Date: Sat, 9 May 2020 11:26:25 +0200 Subject: [PATCH 3/4] Aquire the GIL when calling Python. --- module.c | 9 ++++++++- test_quickjs.py | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/module.c b/module.c index 2df996b..fa6e1f9 100644 --- a/module.c +++ b/module.c @@ -46,14 +46,20 @@ static PyObject *StackOverflow = NULL; // // Takes ownership of the JSValue and will deallocate it (refcount reduced by 1). static PyObject *quickjs_to_python(ContextData *context_obj, JSValue value); +// This method is always called in a context before running JS code in QuickJS. It sets up time +// limites, releases the GIL etc. +static void prepare_call_js(ContextData *context); +// This method is called right after returning from running JS code. Aquires the GIL etc. +static void end_call_js(ContextData *context); static JSModuleDef *js_module_loader(JSContext *ctx, const char *module_name, void *opaque) { ContextData* context = (ContextData*) opaque; JSModuleDef* module = NULL; + + end_call_js(context); if (context->module_loader == NULL || !PyCallable_Check(context->module_loader)) { JS_ThrowReferenceError(ctx, "Could not load module \"%s\": no loader.", module_name); - return NULL; } else { PyObject* result = PyObject_CallFunction(context->module_loader, "s", module_name); @@ -72,6 +78,7 @@ static JSModuleDef *js_module_loader(JSContext *ctx, Py_XDECREF(result); } + prepare_call_js(context); return module; } diff --git a/test_quickjs.py b/test_quickjs.py index 04e0a18..207a898 100644 --- a/test_quickjs.py +++ b/test_quickjs.py @@ -70,6 +70,11 @@ def test_set_module_loader_to_non_callable(self): with self.assertRaises(TypeError): self.context.set_module_loader(2) + def test_import_module_loader_failed(self): + self.context.set_module_loader(lambda: "") + with self.assertRaisesRegex(quickjs.JSException, "loader failed"): + self.context.module('import { foo } from "petter";') + def test_module_does_not_return_code(self): self.context.set_module_loader(lambda name: 1) with self.assertRaisesRegex(quickjs.JSException, "did not return a string"): From fb7bbec931e92578116fb44ef789a133658c7a0a Mon Sep 17 00:00:00 2001 From: Petter Strandmark Date: Sat, 9 May 2020 11:26:58 +0200 Subject: [PATCH 4/4] Format code. --- module.c | 57 +++++++++++++++++++++++++++---------------------- test_quickjs.py | 1 + 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/module.c b/module.c index fa6e1f9..5b5443d 100644 --- a/module.c +++ b/module.c @@ -24,7 +24,7 @@ typedef struct { JSContext *context; int has_time_limit; clock_t time_limit; - PyObject* module_loader; + PyObject *module_loader; // Used when releasing the GIL. PyThreadState *thread_state; InterruptData interrupt_data; @@ -52,34 +52,39 @@ static void prepare_call_js(ContextData *context); // This method is called right after returning from running JS code. Aquires the GIL etc. static void end_call_js(ContextData *context); -static JSModuleDef *js_module_loader(JSContext *ctx, - const char *module_name, void *opaque) { - ContextData* context = (ContextData*) opaque; - JSModuleDef* module = NULL; +static JSModuleDef *js_module_loader(JSContext *ctx, const char *module_name, void *opaque) { + ContextData *context = (ContextData *)opaque; + JSModuleDef *module = NULL; end_call_js(context); if (context->module_loader == NULL || !PyCallable_Check(context->module_loader)) { JS_ThrowReferenceError(ctx, "Could not load module \"%s\": no loader.", module_name); } else { - PyObject* result = PyObject_CallFunction(context->module_loader, "s", module_name); + PyObject *result = PyObject_CallFunction(context->module_loader, "s", module_name); if (result == NULL) { - JS_ThrowReferenceError(ctx, "Could not load module \"%s\": loader failed.", module_name); + JS_ThrowReferenceError( + ctx, "Could not load module \"%s\": loader failed.", module_name); } else if (!PyUnicode_Check(result)) { - JS_ThrowReferenceError(ctx, "Could not load module \"%s\": loader did not return a string.", module_name); + JS_ThrowReferenceError( + ctx, "Could not load module \"%s\": loader did not return a string.", module_name); } else { - const char* code = PyUnicode_AsUTF8(result); - JSValue js_value = JS_Eval(ctx, code, strlen(code), module_name, JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); + const char *code = PyUnicode_AsUTF8(result); + JSValue js_value = JS_Eval(ctx, + code, + strlen(code), + module_name, + JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); if (!JS_IsException(js_value)) { module = JS_VALUE_GET_PTR(js_value); - JS_FreeValue(ctx, js_value); - } + JS_FreeValue(ctx, js_value); + } } Py_XDECREF(result); } prepare_call_js(context); - return module; + return module; } // Returns nonzero if we should stop due to a time limit. @@ -187,13 +192,13 @@ static PyObject *object_get(ObjectData *self, PyObject *args) { // // Returns a list of all properties of a JS object. static PyObject *object_dir(ObjectData *self) { - JSPropertyEnum* props = 0; + JSPropertyEnum *props = 0; uint32_t len; JS_GetOwnPropertyNames(self->context->context, &props, &len, self->object, JS_GPN_STRING_MASK); - PyObject* list = PyList_New(len); - for (int i=0; icontext->context, props[i].atom); + PyObject *list = PyList_New(len); + for (int i = 0; i < len; ++i) { + const char *prop = JS_AtomToCString(self->context->context, props[i].atom); PyList_SetItem(list, i, Py_BuildValue("s", prop)); JS_FreeCString(self->context->context, prop); } @@ -204,8 +209,8 @@ static PyObject *object_dir(ObjectData *self) { // All methods of the _quickjs.Object class. static PyMethodDef object_methods[] = { {"json", (PyCFunction)object_json, METH_NOARGS, "Converts to a JSON string."}, - {"get", (PyCFunction)object_get, METH_VARARGS, "Retrieves a property."}, - {"dir", (PyCFunction)object_dir, METH_NOARGS, "Returns a list of all properties."}, + {"get", (PyCFunction)object_get, METH_VARARGS, "Retrieves a property."}, + {"dir", (PyCFunction)object_dir, METH_NOARGS, "Returns a list of all properties."}, {NULL} /* Sentinel */ }; @@ -392,13 +397,13 @@ static PyObject *context_new(PyTypeObject *type, PyObject *args, PyObject *kwds) // We never have different contexts for the same runtime. This way, different // _quickjs.Context can be used concurrently. self->runtime = JS_NewRuntime(); - self->context = JS_NewContext(self->runtime); + self->context = JS_NewContext(self->runtime); self->has_time_limit = 0; self->time_limit = 0; self->module_loader = NULL; JS_SetModuleLoaderFunc(self->runtime, NULL, js_module_loader, self); - + self->thread_state = NULL; self->python_callables = NULL; JS_SetContextOpaque(self->context, self); @@ -407,7 +412,7 @@ static PyObject *context_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } // Deallocates an instance of the _quickjs.Context class. -static void context_dealloc(ContextData *self) { +static void context_dealloc(ContextData *self) { JS_FreeContext(self->context); JS_FreeRuntime(self->runtime); Py_XDECREF(self->module_loader); @@ -526,7 +531,7 @@ static PyObject *context_set_max_stack_size(ContextData *self, PyObject *args) { // // Sets the max stack size in bytes. static PyObject *context_set_module_loader(ContextData *self, PyObject *args) { - PyObject* loader; + PyObject *loader; if (!PyArg_ParseTuple(args, "O", &loader)) { return NULL; } @@ -709,10 +714,10 @@ static PyMethodDef context_methods[] = { (PyCFunction)context_set_max_stack_size, METH_VARARGS, "Sets the maximum stack size in bytes. Default is 256kB."}, - {"set_module_loader", - (PyCFunction)context_set_module_loader, + {"set_module_loader", + (PyCFunction)context_set_module_loader, METH_VARARGS, - "Sets the Python function that takes a module name and returns its source code."}, + "Sets the Python function that takes a module name and returns its source code."}, {"memory", (PyCFunction)context_memory, METH_NOARGS, "Returns the memory usage as a dict."}, {"gc", (PyCFunction)context_gc, METH_NOARGS, "Runs garbage collection."}, {"add_callable", (PyCFunction)context_add_callable, METH_VARARGS, "Wraps a Python callable."}, diff --git a/test_quickjs.py b/test_quickjs.py index 207a898..5ef3c96 100644 --- a/test_quickjs.py +++ b/test_quickjs.py @@ -272,6 +272,7 @@ def test_time_limit_disallowed(self): with self.assertRaises(quickjs.JSException): self.context.eval("f(40)") + class Object(unittest.TestCase): def setUp(self): self.context = quickjs.Context()