diff --git a/.dockerignore b/.dockerignore index 6aed1bbb..421d19a4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,7 @@ -# Ignore local builds -third-party/llvm-project/build -third-party/wasi-libc/build +.git + +build/ + +# Ignore big third party deps +third-party/llvm-project/ +third-party/wasi-libc/build/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 13fad57c..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Build - -on: - push: - branches: [master] - pull_request: - branches: [master] - -jobs: - formatting: - if: github.event.pull_request.draft == false - runs-on: ubuntu-20.04 - steps: - - name: "Get the latest code" - uses: actions/checkout@v2 - - name: "Install requirements" - run: "pip3 install -r requirements.txt" - - name: "Run Black" - run: "python3 -m black --check $(git ls-files '*.py')" - - name: "Run C/C++ formatting" - run: ./bin/run_clang_format.sh - - name: "Check no formatting changes" - run: git diff --exit-code - - build: - if: github.event.pull_request.draft == false - runs-on: ubuntu-20.04 - steps: - - name: "Get the latest code" - uses: actions/checkout@v2 - - name: "Submodules" - run: "git submodule update --init --recursive" - - name: "Install requirements" - run: "pip3 install -r requirements.txt" - - name: "Set up QEMU" - uses: docker/setup-qemu-action@v1 - - name: "Set up Docker Buildx" - uses: docker/setup-buildx-action@v1 - - name: "Build the sysroot container" - id: docker_build - uses: docker/build-push-action@v2 - with: - push: false - context: . - file: docker/sysroot.dockerfile - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77167f8f..a0590df7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,11 +26,11 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: "Build and push sysroot container" + - name: "Build and push cpp-sysroot container" id: docker_build uses: docker/build-push-action@v2 with: push: true - file: docker/sysroot.dockerfile + file: docker/cpp-sysroot.dockerfile context: . - tags: faasm/sysroot:${{ env.TAG_VERSION }} + tags: faasm/cpp-sysroot:${{ env.TAG_VERSION }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..44c1e422 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,52 @@ +name: Tests + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + formatting: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + steps: + - name: "Get the latest code" + uses: actions/checkout@v2 + - name: "Install requirements" + run: "pip3 install -r requirements.txt" + - name: "Run Black" + run: "python3 -m black --check $(git ls-files '*.py')" + - name: "Run C/C++ formatting" + run: ./bin/run_clang_format.sh + - name: "Check no formatting changes" + run: git diff --exit-code + + tests: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + env: + HOST_TYPE: ci + REDIS_QUEUE_HOST: redis + REDIS_STATE_HOST: redis + container: + image: faasm/cpp-sysroot:0.0.11 + defaults: + run: + working-directory: /code/faasm-toolchain + services: + redis: + image: redis + steps: + - name: "Fetch ref" + run: git fetch origin ${GITHUB_REF}:ci-branch + - name: "Check out branch" + run: git checkout --force ci-branch + - name: "Update Faabric submodule" + run: "git submodule update --init -f third-party/faabric" + - name: "Update CMake build" + run: "inv dev.cmake" + - name: "Build the tests" + run: "inv dev.cc tests" + - name: "Run the tests" + run: "/build/faasm-toolchain/bin/tests" diff --git a/.gitmodules b/.gitmodules index ce798c07..4f34d05a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,3 +17,6 @@ [submodule "third-party/faasm-clapack"] path = third-party/faasm-clapack url = https://github.com/faasm/faasm-clapack +[submodule "third-party/faabric"] + path = third-party/faabric + url = https://github.com/faasm/faabric/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..42a0bdf4 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.13.0) +project(faasm-cpp) + +# Top-level CMake config +set(CMAKE_CXX_FLAGS "-Wall") +set(CMAKE_CXX_FLAGS_DEBUG "-g") +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# Add Faabric dependency +add_subdirectory(third-party/faabric) + +# Generated protobuf headers +include_directories(${CMAKE_CURRENT_BINARY_DIR}/third-party/faabric/src) + +add_subdirectory(libfaasm) +add_subdirectory(emulator) +add_subdirectory(tests) diff --git a/README.md b/README.md index d3e09ac3..06c55747 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -# Faasm Toolchain [![Toolchain tests](https://github.com/faasm/faasm-toolchain/workflows/Build/badge.svg?branch=master)](https://github.com/faasm/faasm-toolchain/actions) [![License](https://img.shields.io/github/license/faasm/faasm-toolchain.svg)](https://github.com/faasm/faasm-toolchain/blob/master/LICENSE.md) +# Faasm C/C++ Support [![CPP tests](https://github.com/faasm/faasm-toolchain/workflows/Tests/badge.svg?branch=master)](https://github.com/faasm/faasm-toolchain/actions) [![License](https://img.shields.io/github/license/faasm/faasm-toolchain.svg)](https://github.com/faasm/faasm-toolchain/blob/master/LICENSE.md) -This is the toolchain and sysroot used for building WebAssembly to run in -[Faasm](https://github.com/faasm/faasm). +This repo contains everything needed to build C/C++ applications for +[Faasm](https://github.com/faasm/faasm): Faasm aims to support a range of legacy applications, so requires a toolchain capable of compiling large projects that may require threading, C++ exceptions @@ -14,10 +14,13 @@ WebAssembly shared libraries outside of the Emscripten target. You can see these in [this fork](https://github.com/faasm/llvm-project) through [this diff](https://github.com/llvm/llvm-project/compare/llvmorg-10.0.1...faasm:faasm). +The repo also contains the C/C++ definition of the [Faasm host +interface](https://github.com/faasm/faasm/blob/master/docs/host_interface.md) +along with an emulator to compile and test native applications. + More detailed docs can be found in: -- [Usage](docs/usage.md) -- [Building and Releasing](docs/release.md) +- [Usage and Development](docs/usage.md) +- [Releasing](docs/release.md) - [Upgrading LLVM](docs/upgrading-llvm.md) -- [Python tooling](docs/python.md) - +- [Python library](docs/python.md) diff --git a/bin/cli.sh b/bin/cli.sh new file mode 100755 index 00000000..47ac68a8 --- /dev/null +++ b/bin/cli.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +THIS_DIR=$(dirname $(readlink -f $0)) +PROJ_ROOT=${THIS_DIR}/.. + +pushd ${PROJ_ROOT} > /dev/null + +if [[ -z "${SYSROOT_CLI_IMAGE}" ]]; then + VERSION=$(cat VERSION) + SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:${VERSION} +fi + +INNER_SHELL=${SHELL:-"/bin/bash"} + +docker-compose \ + up \ + --no-recreate \ + -d \ + cli + +docker-compose \ + exec \ + cli \ + ${INNER_SHELL} + +popd > /dev/null diff --git a/bin/here.py b/bin/here.py index 6ec17c22..2594db73 100644 --- a/bin/here.py +++ b/bin/here.py @@ -8,10 +8,12 @@ def main(): version_file = join(PROJ_ROOT, "VERSION") with open(version_file) as fh: - toolchain_ver = fh.read().strip() + sysroot_ver = fh.read().strip() + + image_tag = "faasm/cpp-sysroot:{}".format(sysroot_ver) cwd = getcwd() - print("Running toolchain at {}".format(cwd)) + print("Running {} at {}".format(image_tag, cwd)) docker_cmd = [ 'docker run --entrypoint="/bin/bash"', @@ -20,7 +22,7 @@ def main(): "-v {}:/work".format(cwd), "-w /work", "-it", - "faasm/sysroot:{}".format(toolchain_ver), + image_tag, ] docker_cmd = " ".join(docker_cmd) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..592a7cbb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +version: "3" + +services: + redis: + image: redis + + cli: + image: ${SYSROOT_CLI_IMAGE} + volumes: + - .:/code/faasm-toolchain + - ./build:/build/faasm-toolchain + working_dir: /code/faasm-toolchain + stdin_open: true + tty: true + privileged: true + environment: + - LOG_LEVEL=debug + - REDIS_STATE_HOST=redis + - REDIS_QUEUE_HOST=redis + depends_on: + - redis diff --git a/docker/cpp-sysroot.dockerfile b/docker/cpp-sysroot.dockerfile new file mode 100644 index 00000000..064447bc --- /dev/null +++ b/docker/cpp-sysroot.dockerfile @@ -0,0 +1,63 @@ +FROM faasm/llvm:10.0.1 as llvm + +FROM faasm/grpc-root:0.0.12 +ARG SYSROOT_VERSION + +# Copy the toolchain in from the LLVM container +COPY --from=llvm /usr/local/faasm /usr/local/faasm + +RUN apt install -y \ + autoconf \ + autotools-dev \ + libtool \ + python3-dev \ + python3-pip + +# Get the code +WORKDIR /code +RUN git clone -b v${SYSROOT_VERSION} https://github.com/faasm/faasm-toolchain +WORKDIR /code/faasm-toolchain + +# Update submodules (not LLVM) +RUN git submodule update --init -f third-party/eigen +RUN git submodule update --init -f third-party/faabric +RUN git submodule update --init -f third-party/faasm-clapack +RUN git submodule update --init -f third-party/libffi +RUN git submodule update --init -f third-party/wasi-libc + +RUN pip3 install -r requirements.txt + +# Install the faasmtools lib +RUN pip3 install . + +# ----------------------------- +# CPP EMULATOR +# ----------------------------- + +RUN inv eigen --native + +RUN inv dev.cmake +RUN inv dev.cc emulator + +# ----------------------------- +# WASM LIBRARIES +# ----------------------------- + +# Install files +RUN inv install + +# Build libraries +RUN inv libc + +# Install eigen to wasm +RUN inv eigen + +# Install libffi +RUN inv libffi + +# Both static and shared clapack +RUN inv clapack +RUN inv clapack --clean --shared + +# Install Faasm CPP wasm lib +RUN inv libfaasm diff --git a/docker/toolchain.dockerfile b/docker/llvm.dockerfile similarity index 100% rename from docker/toolchain.dockerfile rename to docker/llvm.dockerfile diff --git a/docker/sysroot.dockerfile b/docker/sysroot.dockerfile deleted file mode 100644 index 50dc56d0..00000000 --- a/docker/sysroot.dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM faasm/toolchain:10.0.1 - -# Install Python tooling -RUN apt update -RUN apt install -y \ - python3-dev \ - python3-pip \ - libtool \ - autotools-dev \ - autoconf - -WORKDIR /code -COPY requirements.txt . -RUN pip3 install -r requirements.txt - -# Copy the code in -COPY . . - -# Install the faasmtools lib -RUN pip3 install . - -# Install files -RUN inv install - -# Build libraries -RUN inv libc - -RUN inv eigen - -RUN inv libffi - -RUN inv clapack -RUN inv clapack --clean --shared - -RUN inv libfaasm - -# Remove the code -RUN rm -r /code diff --git a/docs/release.md b/docs/release.md index 838e8af1..c7bec2c5 100644 --- a/docs/release.md +++ b/docs/release.md @@ -2,65 +2,48 @@ The toolchain repo is based on two Docker images: -- `faasm/toolchain` - the base image holding just the custom LLVM tools -- `faasm/sysroot` - the image holding both the tools and sysroot +- `faasm/llvm` - the base image holding just the custom LLVM tools +- `faasm/cpp-sysroot` - the image holding both the tools and sysroot See the [Actions page](https://github.com/faasm/faasm-toolchain/actions) and [Dockerfiles](docker) for more info. -You only need to rebuild the `toolchain` image when upgrading LLVM (see +You only need to rebuild the `llvm` image when upgrading LLVM (see [the docs](docs/upgrade-llvm.md). -The `sysroot` image is rebuilt as part of the CI and tagging process. - -## Development - -Tasks related to this repo are run using [Invoke](https://www.pyinvoke.org/). -You just need to set up a virtual environemnt: - -```bash -# Virtualenv -python3 -m venv venv -source venv/bin/activate -pip install requirements.txt - -# List tasks -inv -l -``` +The `cpp-sysroot` image is rebuilt as part of the CI and tagging process. ## Release build -The release build will run the `sysroot` build and push the Docker image to +The release build will run the `cpp-sysroot` build and push the Docker image to Dockerhub. To do this: - Create a branch with your changes -- Update the version in [`VERSION`](../VERSION) +- Update the version in [`VERSION`](../VERSION), `.github/workflows` and `.env` - Push this to your branch - Run `inv git.tag` to create the tag (from the head of the current branch) - Let the CI build run through and build the container -- Once done, create a PR, which will again run through the container build to - check it's ok +- Once done, create a PR, which will run the tests against the new image -## Rebuilding `toolchain` +## Rebuilding `llvm` -You should only need to manually rebuild the `toolchain` image, the `sysroot` +You should only need to manually rebuild the `llvm` image, the `cpp-sysroot` image is built through GH Actions. ```bash -# Build the toolchain image -inv container.toolchain` - rebuilds the base toolchain image +# Build the llvm image +inv container.llvm -# Push the latest toolchain image build -inv container.push-toolchain +# Push it +inv container.push-llvm ``` -## Rebuilding `sysroot` +## Rebuilding `cpp-sysroot` -If you do want to build `sysroot` locally (e.g. for debugging issues): +If you do want to build `cpp-sysroot` locally (e.g. for debugging issues): ```bash inv container.sysroot ``` - diff --git a/docs/usage.md b/docs/usage.md index 4930547b..665a7e73 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1,14 +1,13 @@ # Usage -## Running the `sysroot` container +Everything in this repo is containerised, so to use the tooling you must do so +from within one of the containers. You can also create your own images that +extend those defined in this repo. -The containers built from this repo aren't meant to be used directly, but to -debug and test builds, you _can_ run inside the `sysroot` container with -the `here.py` script. +The [here.py](../bin/here.py) script is a convenience wrapper that will mount +your cwd into the `cpp-sysroot` container at `/work`, e.g. -This will mount your cwd into the sysroot container at `/work`, e.g. - -``` +```bash # Go to some dir cd @@ -20,6 +19,27 @@ python3 /bin/here.py /usr/local/faasm/toolchain/bin/llc --version ``` +## Development + +To develop the project you need to run the built container with your code +mounted in it: + +```bash +cd +./bin/cli.sh + +# List available tasks +inv -l +``` + +You can then compile wasm or build and run the tests, e.g.: + +```bash +inv dev.cmake +inv dev.cc tests +/build/bin/tests +``` + ## Shared libraries For CMake projects, you should be able to add the following to your build: diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt new file mode 100644 index 00000000..7174943a --- /dev/null +++ b/emulator/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 3.0) +project(faasm-emulator) + +set(PUBLIC_HEADERS + faasm/emulator.h + faasm/emulator_api.h + ) + +set(LIB_FILES emulator.cpp) + +add_library(emulator STATIC "${LIB_FILES}") + +target_include_directories(emulator PUBLIC ${CMAKE_CURRENT_LIST_DIR}) + +target_link_libraries(emulator faasm faabric) + +set_target_properties(emulator PROPERTIES PUBLIC_HEADER "${PUBLIC_HEADERS}") + +install(TARGETS emulator + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + PUBLIC_HEADER DESTINATION include/faasm + ) diff --git a/emulator/emulator.cpp b/emulator/emulator.cpp new file mode 100644 index 00000000..3243820c --- /dev/null +++ b/emulator/emulator.cpp @@ -0,0 +1,357 @@ + +extern "C" { +#include +#include +#include +} + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * Used to emulate Faasm in native applications (i.e. anything that's not + * WebAssembly) + * + * This will make use of the Faasm state abstractions where possible. + */ + +// Note thread-locality here +static thread_local faabric::Message _emulatedCall = faabric::Message(); +static thread_local faabric::state::State *_emulatedState = nullptr; + +#define DUMMY_USER "emulated" + +// -------------------------------------------------------------- +// EMULATOR-SPECIFIC +// -------------------------------------------------------------- + +void resetEmulator() { + _emulatedCall = faabric::Message(); + getEmulatorState()->forceClearAll(true); +} + +std::vector getEmulatorOutputData() { + return faabric::util::stringToBytes(_emulatedCall.outputdata()); +} + +std::string getEmulatorOutputDataString() { + const std::vector outputData = getEmulatorOutputData(); + std::string outputStr; + if (outputData.empty()) { + outputStr = "Empty output"; + } else { + outputStr = std::string(reinterpret_cast(outputData.data())); + } + + return outputStr; +} + +std::string getEmulatorUser() { return _emulatedCall.user(); } + +void setEmulatorUser(const char *newUser) { _emulatedCall.set_user(newUser); } + +faabric::state::State *getEmulatorState() { + if (_emulatedState == nullptr) { + return &faabric::state::getGlobalState(); + } else { + return _emulatedState; + } +} + +void setEmulatorState(faabric::state::State *state) { _emulatedState = state; } + +void emulatorSetCallStatus(bool success) { + const std::shared_ptr &logger = faabric::util::getLogger(); + faabric::Message resultMsg = _emulatedCall; + + const std::string funcStr = faabric::util::funcToString(resultMsg, true); + + if (success) { + logger->debug("Setting success status for {}", funcStr); + resultMsg.set_returnvalue(0); + } else { + logger->debug("Setting failed status for {}", funcStr); + resultMsg.set_returnvalue(1); + } + + faabric::scheduler::Scheduler &sch = faabric::scheduler::getScheduler(); + resultMsg.set_outputdata(getEmulatorOutputDataString()); + sch.setFunctionResult(resultMsg); +} + +unsigned int setEmulatedMessageFromJson(const char *messageJson) { + const faabric::Message msg = faabric::util::jsonToMessage(messageJson); + return setEmulatedMessage(msg); +} + +unsigned int setEmulatedMessage(const faabric::Message &msg) { + _emulatedCall = msg; + unsigned int msgId = faabric::util::setMessageId(_emulatedCall); + + const std::shared_ptr &logger = faabric::util::getLogger(); + const std::string funcStr = faabric::util::funcToString(_emulatedCall, true); + logger->debug("Emulator set to {}", funcStr); + + return msgId; +} + +std::string getEmulatedUser() { + if (_emulatedCall.user().empty()) { + const std::shared_ptr &logger = faabric::util::getLogger(); + logger->debug("Setting dummy emulator user {}", DUMMY_USER); + setEmulatorUser(DUMMY_USER); + } + + return _emulatedCall.user(); +} + +// -------------------------------------------------------------- +// FAASM HOST INTERFACE IMPLEMENTATION +// -------------------------------------------------------------- + +std::shared_ptr getKv(const char *key, + size_t size) { + faabric::state::State *s = getEmulatorState(); + const std::string &emulatedUser = getEmulatedUser(); + return s->getKV(emulatedUser, key, size); +} + +std::shared_ptr getKv(const char *key) { + faabric::state::State *s = getEmulatorState(); + const std::string &emulatedUser = getEmulatedUser(); + return s->getKV(emulatedUser, key); +} + +void __faasm_write_output(const unsigned char *output, long outputLen) { + faabric::util::getLogger()->debug("E - write_output {} {}", output, + outputLen); + _emulatedCall.set_outputdata(output, outputLen); +} + +long __faasm_read_state(const char *key, unsigned char *buffer, + long bufferLen) { + faabric::util::getLogger()->debug("E - read_state {} {}", key, bufferLen); + faabric::state::State *s = getEmulatorState(); + std::string emulatedUser = getEmulatedUser(); + + if (bufferLen == 0) { + return s->getStateSize(emulatedUser, key); + } + + auto kv = getKv(key, bufferLen); + kv->get(buffer); + return kv->size(); +} + +unsigned char *__faasm_read_state_ptr(const char *key, long totalLen) { + faabric::util::getLogger()->debug("E - read_state_ptr {} {}", key, totalLen); + auto kv = getKv(key, totalLen); + return kv->get(); +} + +void __faasm_read_state_offset(const char *key, long totalLen, long offset, + unsigned char *buffer, long bufferLen) { + faabric::util::getLogger()->debug("E - read_state_offset {} {} {} {}", key, + totalLen, offset, bufferLen); + auto kv = getKv(key, totalLen); + kv->getChunk(offset, buffer, bufferLen); +} + +unsigned char *__faasm_read_state_offset_ptr(const char *key, long totalLen, + long offset, long len) { + faabric::util::getLogger()->debug("E - read_state_offset_ptr {} {} {} {}", + key, totalLen, offset, len); + auto kv = getKv(key, totalLen); + return kv->getChunk(offset, len); +} + +void __faasm_write_state(const char *key, const uint8_t *data, long dataLen) { + faabric::util::getLogger()->debug("E - write_state {} {}", key, dataLen); + auto kv = getKv(key, dataLen); + kv->set(data); +} + +void __faasm_append_state(const char *key, const uint8_t *data, long dataLen) { + faabric::util::getLogger()->debug("E - append_state {} {}", key, dataLen); + auto kv = getKv(key); + kv->append(data, dataLen); +} + +void __faasm_read_appended_state(const char *key, unsigned char *buffer, + long bufferLen, long nElems) { + faabric::util::getLogger()->debug("E - read_appended_state {} {} {}", key, + bufferLen, nElems); + + const std::string user = getEmulatorUser(); + auto kv = getKv(key); + kv->getAppended(buffer, bufferLen, nElems); +} + +void __faasm_clear_appended_state(const char *key) { + faabric::util::getLogger()->debug("E - clear_appended_state {}", key); + auto kv = getKv(key); + kv->clearAppended(); +} + +void __faasm_write_state_offset(const char *key, long totalLen, long offset, + const unsigned char *data, long dataLen) { + // Avoid excessive logging + // faabric::util::getLogger()->debug("E - write_state_offset {} {} {} {}", + // key, totalLen, offset, dataLen); + auto kv = getKv(key, totalLen); + kv->setChunk(offset, data, dataLen); +} + +unsigned int __faasm_write_state_from_file(const char *key, + const char *filePath) { + faabric::util::getLogger()->debug("E - write_state_from_file - {} {}", key, + filePath); + + // Read file into bytes + const std::vector bytes = faabric::util::readFileToBytes(filePath); + unsigned long fileLength = bytes.size(); + + // Write to state + faabric::state::State *s = getEmulatorState(); + auto kv = s->getKV(getEmulatorUser(), key, fileLength); + kv->set(bytes.data()); + kv->pushFull(); + + return bytes.size(); +} + +void __faasm_flag_state_dirty(const char *key, long totalLen) { + // Avoid excessive logging + // faabric::util::getLogger()->debug("E - flag_state_dirty {} {}", key, + // totalLen); + auto kv = getKv(key, totalLen); + kv->flagDirty(); +} + +void __faasm_flag_state_offset_dirty(const char *key, long totalLen, + long offset, long dataLen) { + // Avoid excessive logging + // faabric::util::getLogger()->debug("E - flag_state_offset_dirty {} {} {} + // {}", key, totalLen, offset, dataLen); + auto kv = getKv(key, totalLen); + kv->flagChunkDirty(offset, dataLen); +} + +void __faasm_push_state_partial_mask(const char *key, const char *maskKey) { + // Avoid excessive logging + // faabric::util::getLogger()->debug("E - push_state_partial_mask {}", key, + // maskKey); + auto kv = getKv(key, 0); + auto maskKv = getKv(maskKey, 0); + + kv->pushPartialMask(maskKv); +} + +void __faasm_push_state(const char *key) { + faabric::util::getLogger()->debug("E - push_state {}", key); + auto kv = getKv(key, 0); + kv->pushFull(); +} + +void __faasm_push_state_partial(const char *key) { + faabric::util::getLogger()->debug("E - push_state_partial {}", key); + auto kv = getKv(key, 0); + kv->pushPartial(); +} + +void __faasm_pull_state(const char *key, long stateLen) { + faabric::util::getLogger()->debug("E - pull_state {}", key); + auto kv = getKv(key, stateLen); + kv->pull(); +} + +long __faasm_read_input(unsigned char *buffer, long bufferLen) { + faabric::util::getLogger()->debug("E - read_input len {}", bufferLen); + + long inputLen; + inputLen = _emulatedCall.inputdata().size(); + + // This relies on thread-local _inputData + if (bufferLen == 0) { + return inputLen; + } + + if (inputLen == 0) { + return 0; + } + + std::copy(_emulatedCall.inputdata().begin(), _emulatedCall.inputdata().end(), + buffer); + + return bufferLen; +} + +unsigned int __faasm_chain_name(const char *name, + const unsigned char *inputData, + long inputDataSize) { + throw std::runtime_error("Chaining not supported in emulator"); +} + +unsigned int __faasm_chain_ptr(int (*funcPtr)(), const unsigned char *buffer, + long bufferLen) { + throw std::runtime_error("Chaining not supported in emulator"); +} + +unsigned int __faasm_chain_py(const char *name, const unsigned char *buffer, + long bufferLen) { + throw std::runtime_error("Chaining not supported in emulator"); +} + +int _await_call_local(unsigned int callId) { + throw std::runtime_error("Chaining not supported in emulator"); +} + +int __faasm_await_call(unsigned int callId) { + throw std::runtime_error("Chaining not supported in emulator"); +} + +int __faasm_await_call_output(unsigned int messageId, unsigned char *buffer, + long bufferLen) { + throw std::runtime_error("Chaining not supported in emulator"); +} + +void __faasm_lock_state_global(const char *key) {} + +void __faasm_unlock_state_global(const char *key) {} + +void __faasm_lock_state_read(const char *key) {} + +void __faasm_unlock_state_read(const char *key) {} + +void __faasm_lock_state_write(const char *key) {} + +void __faasm_unlock_state_write(const char *key) {} + +void copyStringToBuffer(unsigned char *buffer, const std::string &strIn) { + ::strcpy(reinterpret_cast(buffer), strIn.c_str()); +} + +void __faasm_get_py_user(unsigned char *buffer, long bufferLen) { + copyStringToBuffer(buffer, _emulatedCall.pythonuser()); +} + +void __faasm_get_py_func(unsigned char *buffer, long bufferLen) { + copyStringToBuffer(buffer, _emulatedCall.pythonfunction()); +} + +void __faasm_get_py_entry(unsigned char *buffer, long bufferLen) { + copyStringToBuffer(buffer, _emulatedCall.pythonentry()); +} + +unsigned int __faasm_conf_flag(const char *key) { return 0; } diff --git a/emulator/faasm/emulator.h b/emulator/faasm/emulator.h new file mode 100644 index 00000000..73a5a989 --- /dev/null +++ b/emulator/faasm/emulator.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +#include + +unsigned int setEmulatedMessage(const faabric::Message &msg); + +void resetEmulator(); + +void setEmulatorUser(const char *user); + +faabric::state::State *getEmulatorState(); + +void setEmulatorState(faabric::state::State *state); + +std::vector getEmulatorOutputData(); + +std::string getEmulatorOutputDataString(); + +std::string getEmulatorUser(); diff --git a/emulator/faasm/emulator_api.h b/emulator/faasm/emulator_api.h new file mode 100644 index 00000000..d446c808 --- /dev/null +++ b/emulator/faasm/emulator_api.h @@ -0,0 +1,8 @@ +#ifndef FAASM_EMULATOR_API_H +#define FAASM_EMULATOR_API_H + +unsigned int setEmulatedMessageFromJson(const char *msgJson); + +void emulatorSetCallStatus(bool success); + +#endif diff --git a/faasmtools/env.py b/faasmtools/env.py index 14f29f66..0f2809a1 100644 --- a/faasmtools/env.py +++ b/faasmtools/env.py @@ -4,12 +4,6 @@ PROJ_ROOT = dirname(dirname(abspath(__file__))) THIRD_PARTY_DIR = join(PROJ_ROOT, "third-party") -# Docker -TOOLCHAIN_IMAGE_NAME = "faasm/toolchain" -TOOLCHAIN_DOCKERFILE = join(PROJ_ROOT, "docker", "toolchain.dockerfile") -SYSROOT_IMAGE_NAME = "faasm/sysroot" -SYSROOT_DOCKERFILE = join(PROJ_ROOT, "docker", "sysroot.dockerfile") - # Environment USABLE_CPUS = int(cpu_count()) - 1 @@ -23,12 +17,3 @@ def get_version(): ver = fh.read() return ver.strip() - - -def get_sysroot_tag(): - version = get_version() - return "{}:{}".format(SYSROOT_IMAGE_NAME, version) - - -def get_toolchain_tag(): - return "{}:{}".format(TOOLCHAIN_IMAGE_NAME, LLVM_VERSION) diff --git a/faasmtools/git.py b/faasmtools/git.py index 6f73da22..7d08af56 100644 --- a/faasmtools/git.py +++ b/faasmtools/git.py @@ -1,11 +1,16 @@ from subprocess import run -def tag_project(tag_name, proj_dir): - run("git tag {}".format(tag_name), shell=True, check=True, cwd=proj_dir) +def tag_project(tag_name, proj_dir, force=False): + run( + "git tag {} {}".format("--force" if force else "", tag_name), + shell=True, + check=True, + cwd=proj_dir, + ) run( - "git push origin {}".format(tag_name), + "git push {} origin {}".format("--force" if force else "", tag_name), shell=True, check=True, cwd=proj_dir, diff --git a/libfaasm/CMakeLists.txt b/libfaasm/CMakeLists.txt index 6a00387a..5de4a9d9 100644 --- a/libfaasm/CMakeLists.txt +++ b/libfaasm/CMakeLists.txt @@ -5,8 +5,6 @@ project(libfaasm) set(CMAKE_CXX_STANDARD 17) -include_directories(${CMAKE_CURRENT_LIST_DIR}) - set(PUBLIC_HEADERS faasm/core.h faasm/host_interface.h @@ -62,9 +60,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Wasm") else () message(STATUS "Libfaasm native build") - # Note - this should be a shared library to support linking with external - # projects - add_library(faasm SHARED "${LIB_FILES}") + add_library(faasm STATIC "${LIB_FILES}") set_target_properties(faasm PROPERTIES PUBLIC_HEADER "${PUBLIC_HEADERS}") @@ -74,3 +70,5 @@ else () PUBLIC_HEADER DESTINATION include/faasm ) endif () + +target_include_directories(faasm PUBLIC ${CMAKE_CURRENT_LIST_DIR}) diff --git a/tasks/__init__.py b/tasks/__init__.py index d78acce4..d3f8eb3d 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -2,6 +2,7 @@ from . import clapack from . import container +from . import dev from . import eigen from . import git from . import install @@ -12,6 +13,7 @@ ns = Collection( clapack, container, + dev, eigen, git, install, diff --git a/tasks/container.py b/tasks/container.py index 6909cb9a..720d21e7 100644 --- a/tasks/container.py +++ b/tasks/container.py @@ -1,27 +1,41 @@ from invoke import task +from os.path import join + from faasmtools.docker import ( build_container, push_container, ) from faasmtools.env import ( - get_toolchain_tag, - get_sysroot_tag, + get_version, PROJ_ROOT, - TOOLCHAIN_DOCKERFILE, - SYSROOT_DOCKERFILE, + LLVM_VERSION, ) +LLVM_IMAGE_NAME = "faasm/llvm" +LLVM_DOCKERFILE = join(PROJ_ROOT, "docker", "llvm.dockerfile") +SYSROOT_IMAGE_NAME = "faasm/cpp-sysroot" +SYSROOT_DOCKERFILE = join(PROJ_ROOT, "docker", "cpp-sysroot.dockerfile") + + +def get_sysroot_tag(): + version = get_version() + return "{}:{}".format(SYSROOT_IMAGE_NAME, version) + + +def get_llvm_tag(): + return "{}:{}".format(LLVM_IMAGE_NAME, LLVM_VERSION) + @task -def toolchain(ctx, nocache=False, push=False): +def llvm(ctx, nocache=False, push=False): """ - Build current version of the toolchain container + Build current version of the llvm container """ build_container( - get_toolchain_tag(), - TOOLCHAIN_DOCKERFILE, + get_llvm_tag(), + LLVM_DOCKERFILE, PROJ_ROOT, nocache=nocache, push=push, @@ -29,11 +43,11 @@ def toolchain(ctx, nocache=False, push=False): @task -def push_toolchain(ctx): +def push_llvm(ctx): """ - Push the current version of the toolchain container + Push the current version of the llvm container """ - push_container(get_toolchain_tag()) + push_container(get_llvm_tag()) @task @@ -47,6 +61,7 @@ def sysroot(ctx, nocache=False, push=False): PROJ_ROOT, nocache=nocache, push=push, + build_args={"SYSROOT_VERSION": get_version()}, ) diff --git a/tasks/dev.py b/tasks/dev.py new file mode 100644 index 00000000..f438a60d --- /dev/null +++ b/tasks/dev.py @@ -0,0 +1,56 @@ +from os import makedirs +from os.path import exists +from subprocess import run +from shutil import rmtree + +from invoke import task + +from faasmtools.env import PROJ_ROOT + +BUILD_DIR = "/build/faasm-toolchain" + + +@task +def cmake(ctx, clean=False, build="Debug"): + """ + Configures the CMake build + """ + if clean and exists(BUILD_DIR): + rmtree(BUILD_DIR) + + if not exists(BUILD_DIR): + makedirs(BUILD_DIR) + + cmd = [ + "cmake", + "-GNinja", + "-DCMAKE_BUILD_TYPE={}".format(build), + "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-10", + "-DCMAKE_C_COMPILER=/usr/bin/clang-10", + PROJ_ROOT, + ] + + cmd_str = " ".join(cmd) + print(cmd_str) + run(cmd_str, shell=True, check=True, cwd=BUILD_DIR) + + +@task +def cc(ctx, target, clean=False): + """ + Compiles the given CMake target + """ + if clean: + cmake(ctx, clean=True) + + if target == "all": + target = "" + else: + target = "--target {}".format(target) + + run( + "cmake --build . {}".format(target), + cwd=BUILD_DIR, + shell=True, + check=True, + ) diff --git a/tasks/eigen.py b/tasks/eigen.py index d2362bca..ad6cbfbd 100644 --- a/tasks/eigen.py +++ b/tasks/eigen.py @@ -17,12 +17,16 @@ @task(default=True) -def eigen(ctx, verbose=False): +def eigen(ctx, verbose=False, native=False): """ Compile and install Eigen """ work_dir = join(THIRD_PARTY_DIR, "eigen") - build_dir = join(work_dir, "build") + + if native: + build_dir = join(work_dir, "build", "native") + else: + build_dir = join(work_dir, "build", "wasm") if exists(build_dir): rmtree(build_dir) @@ -30,17 +34,24 @@ def eigen(ctx, verbose=False): verbose_string = "VERBOSE=1" if verbose else "" - cmd = [ + cmake_cmd = [ verbose_string, "cmake", "-GNinja", - "-DCMAKE_TOOLCHAIN_FILE={}".format(CMAKE_TOOLCHAIN_FILE), - "-DCMAKE_INSTALL_PREFIX={}".format(WASM_SYSROOT), - work_dir, ] - cmd_string = " ".join(cmd) - run(cmd_string, shell=True, cwd=build_dir, check=True) + if not native: + cmake_cmd.extend( + [ + "-DCMAKE_TOOLCHAIN_FILE={}".format(CMAKE_TOOLCHAIN_FILE), + "-DCMAKE_INSTALL_PREFIX={}".format(WASM_SYSROOT), + ] + ) + + cmake_cmd.append(work_dir) + + cmake_cmd = " ".join(cmake_cmd) + run(cmake_cmd, shell=True, cwd=build_dir, check=True) run( "{} ninja install".format(verbose_string), diff --git a/tasks/git.py b/tasks/git.py index 53905662..9d2dba1d 100644 --- a/tasks/git.py +++ b/tasks/git.py @@ -5,10 +5,10 @@ @task -def tag(ctx): +def tag(ctx, force=False): """ Creates git tag from the current tree """ version = get_version() tag_name = "v{}".format(version) - tag_project(tag_name, PROJ_ROOT) + tag_project(tag_name, PROJ_ROOT, force=force) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 00000000..f3fb65f0 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,12 @@ +file(GLOB_RECURSE TEST_FILES ${CMAKE_CURRENT_LIST_DIR} test_*.cpp) + +add_executable( + tests + main.cpp + ${TEST_FILES} +) + +target_link_libraries(tests emulator faasm faabric_test_utils) + +add_test(NAME tests COMMAND "tests/test/tests") + diff --git a/tests/main.cpp b/tests/main.cpp new file mode 100644 index 00000000..00bd710d --- /dev/null +++ b/tests/main.cpp @@ -0,0 +1,14 @@ +#define CATCH_CONFIG_RUNNER + +#include +#include + +int main(int argc, char *argv[]) { + faabric::util::initLogging(); + + int result = Catch::Session().run(argc, argv); + + fflush(stdout); + + return result; +} diff --git a/tests/test_emulator.cpp b/tests/test_emulator.cpp new file mode 100644 index 00000000..fca4de4c --- /dev/null +++ b/tests/test_emulator.cpp @@ -0,0 +1,75 @@ +#include + +extern "C" { +#include +#include +} + +#include +#include +#include +#include +#include + +namespace tests { + +TEST_CASE("Test emulation", "[emulator]") { + faabric::state::getGlobalState().forceClearAll(true); + + std::vector dummyBytes = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + long dummyLen = dummyBytes.size(); + + faabric::Message call = faabric::util::messageFactory("demo", "echo"); + + SECTION("Output data") { + setEmulatedMessage(call); + faasmSetOutput(dummyBytes.data(), dummyLen); + const std::vector actual = getEmulatorOutputData(); + REQUIRE(actual == dummyBytes); + } + + SECTION("Input data") { + call.set_inputdata(dummyBytes.data(), dummyBytes.size()); + setEmulatedMessage(call); + + long actual = faasmGetInputSize(); + REQUIRE(actual == dummyLen); + + std::vector actualBytes(dummyLen); + faasmGetInput(actualBytes.data(), dummyLen); + + REQUIRE(actualBytes == dummyBytes); + } + + SECTION("Read/ write state") { + setEmulatedMessage(call); + + std::string key = "foobar"; + faasmWriteState(key.c_str(), dummyBytes.data(), dummyLen); + faasmPushState(key.c_str()); + + // Check reading from state + std::vector actual(dummyLen); + faasmReadState(key.c_str(), actual.data(), dummyLen); + REQUIRE(actual == dummyBytes); + } + + SECTION("Read/ write state offset") { + setEmulatedMessage(call); + + std::string key = "foobar_off"; + std::vector offsetData = {7, 6, 5}; + long offset = 2; + long dataLen = 3; + faasmWriteStateOffset(key.c_str(), dummyLen, offset, offsetData.data(), + dataLen); + faasmPushState(key.c_str()); + + // Check reading from state + std::vector actual(dataLen); + faasmPullState(key.c_str(), dummyLen); + faasmReadStateOffset(key.c_str(), dummyLen, offset, actual.data(), dataLen); + REQUIRE(actual == offsetData); + } +} +} // namespace tests diff --git a/third-party/faabric b/third-party/faabric new file mode 160000 index 00000000..3e888628 --- /dev/null +++ b/third-party/faabric @@ -0,0 +1 @@ +Subproject commit 3e8886281dca4c020bf943fa39fc3ebbd30607f6