Skip to content

Commit

Permalink
Merge pull request #274 from Distributive-Network/wes/DCP-4054/make-d…
Browse files Browse the repository at this point in the history 
…cp-client-grok-pythonmonkey

dcp-client compatibility
  • Loading branch information
@philippedistributive
philippedistributive authored Apr 4, 2024
2 parents e9195bc + 1d13bff commit 9f52f58
Show file tree
Hide file tree
Showing 8 changed files with 201 additions and 59 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
include(FetchContent)

SET(COMPILE_FLAGS "-ggdb -Ofast -fno-rtti") # optimize but also emit debug symbols
SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILE_FLAGS}" )
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILE_FLAGS} $ENV{EXTRA_CMAKE_CXX_FLAGS}")

set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
if(APPLE)
Expand Down
46 changes: 46 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# @file Makefile
# Not part of the PythonMonkey build - just workflow helper for Wes.
# @author Wes Garland, [email protected]
# @date March 2024
#

BUILD = debug
PYTHON = python3
RUN = poetry run

PYTHON_BUILD_ENV = VERBOSE=1 EXTRA_CMAKE_CXX_FLAGS="$(EXTRA_CMAKE_CXX_FLAGS)"
OS_NAME := $(shell uname -s)

ifeq ($(OS_NAME),Linux)
CPU_COUNT=$(shell cat /proc/cpuinfo | grep -c processor)
MAX_JOBS=10
CPUS := $(shell test $(CPU_COUNT) -lt $(MAX_JOBS) && echo $(CPU_COUNT) || echo $(MAX_JOBS))
PYTHON_BUILD_ENV += CPUS=$(CPUS)
endif

EXTRA_CMAKE_CXX_FLAGS = -Wno-invalid-offsetof $(JOBS)

ifeq ($(BUILD),debug)
EXTRA_CMAKE_CXX_FLAGS += -O0
endif

.PHONY: build test all clean debug
build:
$(PYTHON_BUILD_ENV) $(PYTHON) ./build.py

test:
$(RUN) ./peter-jr tests
$(RUN) pytest tests/python

all: build test

clean:
rm -rf build/src/CMakeFiles/pythonmonkey.dir
rm -f build/src/pythonmonkey.so
rm -f python/pythonmonkey.so

debug:
@echo EXTRA_CMAKE_CXX_FLAGS=$(EXTRA_CMAKE_CXX_FLAGS)
@echo JOBS=$(JOBS)
@echo CPU_COUNT=$(CPU_COUNT)
@echo OS_NAME=$(OS_NAME)
2 changes: 1 addition & 1 deletion build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
BUILD_DIR = os.path.join(TOP_DIR, "build")

# Get number of CPU cores
CPUS = os.cpu_count() or 1
CPUS = os.getenv('CPUS') or os.cpu_count() or 1

def execute(cmd: str, cwd: Optional[str] = None):
popen = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
Expand Down
12 changes: 9 additions & 3 deletions python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,36 +42,44 @@ async def request(
onNetworkError: Callable[[aiohttp.ClientError], None],
/
):
debug = pm.bootstrap.require("debug");

class BytesPayloadWithProgress(aiohttp.BytesPayload):
_chunkMaxLength = 2**16 # aiohttp default

async def write(self, writer) -> None:
debug('xhr:io')('begin chunked write')
buf = io.BytesIO(self._value)
chunk = buf.read(self._chunkMaxLength)
while chunk:
debug('xhr:io')(' writing', len(chunk), 'bytes')
await writer.write(chunk)
processRequestBodyChunkLength(len(chunk))
chunk = buf.read(self._chunkMaxLength)
processRequestEndOfBody()
debug('xhr:io')('finish chunked write')

if isinstance(body, str):
body = bytes(body, "utf-8")

# set default headers
headers.setdefault("user-agent", f"Python/{platform.python_version()} PythonMonkey/{pm.__version__}")
debug('xhr:headers')('after set default\n', headers)

if timeoutMs > 0:
timeoutOptions = aiohttp.ClientTimeout(total=timeoutMs/1000) # convert to seconds
else:
timeoutOptions = aiohttp.ClientTimeout() # default timeout

try:
debug('xhr:aiohttp')('creating request for', url)
async with aiohttp.request(method=method,
url=yarl.URL(url, encoded=True),
headers=headers,
data=BytesPayloadWithProgress(body) if body else None,
timeout=timeoutOptions,
) as res:
debug('xhr:aiohttp')('got', res.content_type, 'result')
def getResponseHeader(name: str):
return res.headers.get(name)
def getAllResponseHeaders():
Expand All @@ -81,6 +89,7 @@ def getAllResponseHeaders():
headers.sort()
return "\r\n".join(headers)
def abort():
debug('xhr:io')('abort')
res.close()

# readyState HEADERS_RECEIVED
Expand All @@ -92,15 +101,12 @@ def abort():
'getResponseHeader': getResponseHeader,
'getAllResponseHeaders': getAllResponseHeaders,
'abort': abort,

'contentLength': res.content_length or 0,
}
processResponse(responseData)

# readyState LOADING
async for data in res.content.iter_any():
processBodyChunk(bytearray(data)) # PythonMonkey only accepts the mutable bytearray type

# readyState DONE
processEndOfBody()
except asyncio.TimeoutError as e:
Expand Down
55 changes: 50 additions & 5 deletions python/pythonmonkey/builtin_modules/XMLHttpRequest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,39 @@
*
* @copyright Copyright (c) 2023 Distributive Corp.
*/
'use strict';

const { EventTarget, Event } = require('event-target');
const { DOMException } = require('dom-exception');
const { URL, URLSearchParams } = require('url');
const { request, decodeStr } = require('XMLHttpRequest-internal');
const debug = globalThis.python.eval('__import__("pythonmonkey").bootstrap.require')('debug');

/**
* Truncate a string-like thing for display purposes, returning a string.
* @param {any} what The thing to truncate; must have a slice method and index property.
* Works with string, array, typedarray, etc.
* @param {number} maxlen The maximum length for truncation
* @param {boolean} coerce Not false = coerce to printable character codes
* @returns {string}
*/
function trunc(what, maxlen, coerce)
{
if (coerce !== false && typeof what !== 'string')
{
what = Array.from(what).map(x => {
if (x > 31 && x < 127)
return String.fromCharCode(x);
else if (x < 32)
return String.fromCharCode(0x2400 + Number(x));
else if (x === 127)
return '\u2421';
else
return '\u2423';
}).join('');
}
return `${what.slice(0, maxlen)}${what.length > maxlen ? '\u2026' : ''}`;
}

// exposed
/**
Expand All @@ -29,6 +57,7 @@ class ProgressEvent extends Event
this.lengthComputable = eventInitDict.lengthComputable ?? false;
this.loaded = eventInitDict.loaded ?? 0;
this.total = eventInitDict.total ?? 0;
this.debugTag = 'xhr:';
}
}

Expand Down Expand Up @@ -112,6 +141,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
*/
open(method, url, async = true, username = null, password = null)
{
debug('xhr:open')('open start, method=' + method);
// Normalize the method.
// @ts-expect-error
method = method.toString().toUpperCase();
Expand All @@ -125,7 +155,8 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
parsedURL.username = username;
if (password)
parsedURL.password = password;

debug('xhr:open')('url is ' + parsedURL.href);

// step 11
this.#sendFlag = false;
this.#uploadListenerFlag = false;
Expand All @@ -144,6 +175,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
this.#state = XMLHttpRequest.OPENED;
this.dispatchEvent(new Event('readystatechange'));
}
debug('xhr:open')('finished open, state is ' + this.#state);
}

/**
Expand All @@ -153,6 +185,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
*/
setRequestHeader(name, value)
{
debug('xhr:headers')(`set header ${name}=${value}`);
if (this.#state !== XMLHttpRequest.OPENED)
throw new DOMException('setRequestHeader can only be called when state is OPEN', 'InvalidStateError');
if (this.#sendFlag)
Expand Down Expand Up @@ -218,6 +251,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
*/
send(body = null)
{
debug('xhr:send')(`sending; body length=${body?.length}`);
if (this.#state !== XMLHttpRequest.OPENED) // step 1
throw new DOMException('connection must be opened before send() is called', 'InvalidStateError');
if (this.#sendFlag) // step 2
Expand Down Expand Up @@ -248,10 +282,9 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget

const originalAuthorContentType = this.#requestHeaders['content-type'];
if (!originalAuthorContentType && extractedContentType)
{
this.#requestHeaders['content-type'] = extractedContentType;
}
}
debug('xhr:send')(`content-type=${this.#requestHeaders['content-type']}`);

// step 5
if (this.#uploadObject._hasAnyListeners())
Expand All @@ -276,6 +309,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
*/
#sendAsync()
{
debug('xhr:send')('sending in async mode');
this.dispatchEvent(new ProgressEvent('loadstart', { loaded:0, total:0 })); // step 11.1

let requestBodyTransmitted = 0; // step 11.2
Expand Down Expand Up @@ -308,6 +342,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
let responseLength = 0;
const processResponse = (response) =>
{
debug('xhr:response')(`response headers ----\n${response.getAllResponseHeaders()}`);
this.#response = response; // step 11.9.1
this.#state = XMLHttpRequest.HEADERS_RECEIVED; // step 11.9.4
this.dispatchEvent(new Event('readystatechange')); // step 11.9.5
Expand All @@ -318,6 +353,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget

const processBodyChunk = (/** @type {Uint8Array} */ bytes) =>
{
debug('xhr:response')(`recv chunk, ${bytes.length} bytes (${trunc(bytes, 100)})`);
this.#receivedBytes.push(bytes);
if (this.#state === XMLHttpRequest.HEADERS_RECEIVED)
this.#state = XMLHttpRequest.LOADING;
Expand All @@ -330,16 +366,22 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
*/
const processEndOfBody = () =>
{
debug('xhr:response')(`end of body, received ${this.#receivedLength} bytes`);
const transmitted = this.#receivedLength; // step 3
const length = responseLength || 0; // step 4

this.dispatchEvent(new ProgressEvent('progress', { loaded:transmitted, total:length })); // step 6
this.#state = XMLHttpRequest.DONE; // step 7
this.#sendFlag = false; // step 8

this.dispatchEvent(new Event('readystatechange')); // step 9
for (const eventType of ['load', 'loadend']) // step 10, step 11
this.dispatchEvent(new ProgressEvent(eventType, { loaded:transmitted, total:length }));
};

debug('xhr:send')(`${this.#requestMethod} ${this.#requestURL.href}`);
debug('xhr:headers')('headers=' + Object.entries(this.#requestHeaders));

// send() step 6
request(
this.#requestMethod,
Expand All @@ -362,8 +404,8 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
*/
#sendSync()
{
/* Synchronous XHR deprecated. /wg march 2024 */
throw new DOMException('synchronous XHR is not supported', 'NotSupportedError');
// TODO: handle synchronous request
}

/**
Expand All @@ -376,7 +418,6 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
return;
if (this.#timedOutFlag) // step 2
return this.#reportRequestError('timeout', new DOMException(e.toString(), 'TimeoutError'));
console.error(e); // similar to browsers, print out network errors even then the error will be handled by `xhr.onerror`
if (this.#response === null /* network error */) // step 4
return this.#reportRequestError('error', new DOMException(e.toString(), 'NetworkError'));
else // unknown errors
Expand Down Expand Up @@ -652,6 +693,10 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
}
}

/* A side-effect of loading this module is to add the XMLHttpRequest and related symbols to the global
* object. This makes them accessible in the "normal" way (like in a browser) even in PythonMonkey JS
* host environments which don't include a require() symbol.
*/
if (!globalThis.XMLHttpRequestEventTarget)
globalThis.XMLHttpRequestEventTarget = XMLHttpRequestEventTarget;
if (!globalThis.XMLHttpRequestUpload)
Expand Down
Loading

0 comments on commit 9f52f58

Please sign in to comment.