diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7e310ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +target/ +result +*.dia~ +docker-image-hexstody.tar.gz \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..5169d59 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] + +members = [ + "hexstody-hot", + "hexstody-db", +] \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cf8a4c6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM rust:1 as builder +RUN apt-get update && apt-get install -y clang postgresql sudo +COPY . . +RUN ./docker/setup-build-postgresql.sh + +FROM debian:bullseye-slim +COPY --from=builder target/release/hexstody-hot /hexstody-hot +VOLUME /data +WORKDIR /data +RUN apt update && apt install -y libssl1.1 ca-certificates && rm -rf /var/lib/apt/lists/* +COPY ./docker/wait-for-it.sh /wait-for-it.sh diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..b341ae5 --- /dev/null +++ b/default.nix @@ -0,0 +1,42 @@ +let + sources = import ./nix/sources.nix; + nixpkgs-mozilla = import sources.nixpkgs-mozilla; + pkgs = import sources.nixpkgs { + overlays = + [ + nixpkgs-mozilla + (self: super: + let chan = self.rustChannelOf { date = "2022-01-25"; channel = "nightly"; }; + in { + rustc = chan.rust; + cargo = chan.rust; + } + ) + ]; + }; + naersk = pkgs.callPackage sources.naersk {}; + merged-openssl = pkgs.symlinkJoin { name = "merged-openssl"; paths = [ pkgs.openssl.out pkgs.openssl.dev ]; }; +in +naersk.buildPackage { + name = "hexstody"; + root = pkgs.lib.sourceFilesBySuffices ./. [".rs" ".toml" ".lock" ".html" ".css" ".png" ".sh" ".sql"]; + buildInputs = with pkgs; [ openssl pkgconfig clang llvm llvmPackages.libclang zlib cacert curl postgresql ]; + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang}/lib"; + OPENSSL_DIR = "${merged-openssl}"; + preBuild = '' + echo "Deploying local PostgreSQL" + initdb ./pgsql-data --auth=trust + echo "unix_socket_directories = '$PWD'" >> ./pgsql-data/postgresql.conf + pg_ctl start -D./pgsql-data -l psqlog + psql --host=$PWD -d postgres -c "create role \"hexstody\" with login password 'hexstody';" + psql --host=$PWD -d postgres -c "create database \"hexstody\" owner \"hexstody\";" + cp -r ${./hexstody-db/migrations} ./hexstody-db/migrations + for f in ./hexstody-db/migrations/*.sql + do + echo "Applying $f" + psql --host=$PWD -U hexstody -d hexstody -f $f + done + export DATABASE_URL=postgres://hexstody:hexstody@localhost/hexstody + echo "Local database accessible by $DATABASE_URL" + ''; +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..84bf49b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3.9" +services: + postgres: + image: postgres:latest + environment: + - POSTGRES_PASSWORD=hexstody + - POSTGRES_USER=hexstody + - POSTGRES_DB=hexstody + + service: + image: hexstody:latest + ports: + - "8081:8081" + hostname: kolliderhedge + environment: + - HEXSTODY_POSTGRES=postgres://hexstody:hexstody@postgres:5432/hexstody + - HEXSTODY_PORT=8081 + - RUST_LOG=hexstody::api,hexstody=debug,hexstody_domain=debug + # Set it via .env file + - HEXSTODY_API_KEY=$HEXSTODY_API_KEY + command: /wait-for-it.sh postgres:5432 -- /hexstody-hot serve + links: + - postgres \ No newline at end of file diff --git a/docker/setup-build-postgresql.sh b/docker/setup-build-postgresql.sh new file mode 100755 index 0000000..a0cf9cc --- /dev/null +++ b/docker/setup-build-postgresql.sh @@ -0,0 +1,14 @@ +# Designed for debian +echo "Deploying local PostgreSQL" +pg_ctlcluster 13 main start +sudo -u postgres psql -d postgres -c "create role \"hexstody\" with login password 'hexstody';" +sudo -u postgres psql -d postgres -c "create database \"hexstody\" owner \"hexstody\";" +for f in ./hexstody-hedge-db/migrations/*.sql +do + echo "Applying $f" + sudo -u postgres psql -d hexstody -f $f +done +sudo -u postgres psql -d hexstody -c "GRANT ALL PRIVILEGES ON TABLE updates TO hexstody;" +export DATABASE_URL=postgres://hexstody:hexstody@localhost/hexstody +echo "Local database accessible by $DATABASE_URL" +cargo build --release \ No newline at end of file diff --git a/docker/wait-for-it.sh b/docker/wait-for-it.sh new file mode 100755 index 0000000..3974640 --- /dev/null +++ b/docker/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi \ No newline at end of file diff --git a/docs/diagrams/hotwallet_components.dia b/docs/diagrams/hotwallet_components.dia new file mode 100644 index 0000000..4e22014 Binary files /dev/null and b/docs/diagrams/hotwallet_components.dia differ diff --git a/docs/diagrams/hotwallet_components.png b/docs/diagrams/hotwallet_components.png new file mode 100644 index 0000000..de51611 Binary files /dev/null and b/docs/diagrams/hotwallet_components.png differ diff --git a/hexstody-db/Cargo.toml b/hexstody-db/Cargo.toml new file mode 100644 index 0000000..10ef219 --- /dev/null +++ b/hexstody-db/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "hexstody-db" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = { version = "3.0.0-rc.4", features = ["derive", "env"] } +dotenv = "0.15.0" +env_logger = { version = "0.9.0" } +log = "0.4.14" +sqlx = { version = "0.5", features = [ "runtime-tokio-rustls", "migrate", "macros", "postgres", "json", "chrono" ] } +tokio = { version = "1", features = ["full"] } \ No newline at end of file diff --git a/hexstody-db/build.rs b/hexstody-db/build.rs new file mode 100644 index 0000000..3a8149e --- /dev/null +++ b/hexstody-db/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rerun-if-changed=migrations"); +} diff --git a/hexstody-db/migrations/0001_create_scheme.sql b/hexstody-db/migrations/0001_create_scheme.sql new file mode 100644 index 0000000..e69de29 diff --git a/hexstody-db/src/main.rs b/hexstody-db/src/main.rs new file mode 100644 index 0000000..b57f348 --- /dev/null +++ b/hexstody-db/src/main.rs @@ -0,0 +1,47 @@ +use clap::Parser; +use std::error::Error; +use sqlx::postgres::PgPoolOptions; +use log::*; + +#[derive(Parser, Debug)] +#[clap(about, version, author)] +struct Args { + /// PostgreSQL connection string + #[clap( + long, + short, + default_value = "postgres://hexstody:hexstody@localhost/hexstody", + env = "HEXSTODY_POSTGRES" + )] + dbconnect: String, + #[clap(subcommand)] + subcmd: SubCommand, +} + +#[derive(Parser, Debug)] +enum SubCommand { + /// Apply migrations to the given database + Migrate, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let args = Args::parse(); + + env_logger::init(); + + match args.subcmd { + SubCommand::Migrate => { + info!("Connecting to database"); + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(&args.dbconnect) + .await?; + + info!("Applying migrations"); + sqlx::migrate!("./migrations").run(&pool).await?; + info!("Done"); + } + } + Ok(()) +} diff --git a/hexstody-hot/Cargo.toml b/hexstody-hot/Cargo.toml new file mode 100644 index 0000000..354357f --- /dev/null +++ b/hexstody-hot/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "hexstody-hot" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +chrono = { version = "0.4.19", features = [ "serde" ] } +clap = { version = "3.0.0-rc.4", features = ["derive", "env"] } +dotenv = "0.15.0" +env_logger = { version = "0.9.0" } +futures = "0.3.19" +futures-channel = "0.3" +futures-util = "0.3.19" +log = "0.4.14" +rweb = { version = "0.15.0", features = ["openapi", "chrono"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +sqlx = { version = "0.5", features = [ "runtime-tokio-rustls", "migrate", "macros", "postgres", "json", "chrono" ] } +thiserror = "1.0" +tokio = { version = "1", features = ["full"] } +uuid = { version = "0.8.2", features = ["v4"]} + +[dev-dependencies] +maplit = "1.0.2" +sqlx-database-tester = { version = "0.2.0", features = [ "runtime-tokio" ] } \ No newline at end of file diff --git a/hexstody-hot/src/api/mod.rs b/hexstody-hot/src/api/mod.rs new file mode 100644 index 0000000..3244898 --- /dev/null +++ b/hexstody-hot/src/api/mod.rs @@ -0,0 +1 @@ +pub mod public; \ No newline at end of file diff --git a/hexstody-hot/src/api/public.rs b/hexstody-hot/src/api/public.rs new file mode 100644 index 0000000..e69de29 diff --git a/hexstody-hot/src/db.rs b/hexstody-hot/src/db.rs new file mode 100644 index 0000000..e69de29 diff --git a/hexstody-hot/src/main.rs b/hexstody-hot/src/main.rs new file mode 100644 index 0000000..5862871 --- /dev/null +++ b/hexstody-hot/src/main.rs @@ -0,0 +1,87 @@ +mod api; +mod db; + +#[cfg(test)] +#[macro_use] +extern crate maplit; + +use clap::Parser; +use log::*; +use std::error::Error; +use std::time::Duration; +use tokio::time::{sleep, timeout}; + +#[derive(Parser, Debug, Clone)] +#[clap(about, version, author)] +struct Args { + // #[clap(long, env = "KOLLIDER_API_KEY", hide_env_values = true)] + // api_key: String, + /// PostgreSQL connection string + #[clap( + long, + short, + default_value = "postgres://hexstody:hexstody@localhost/hexstody", + env = "HEXSTODY_POSTGRES" + )] + dbconnect: String, + #[clap(subcommand)] + subcmd: SubCommand, +} + +#[derive(Parser, Debug, Clone)] +enum SubCommand { + /// Start listening incoming API requests + Serve { + /// Host name to bind the service to + #[clap( + long, + default_value = "0.0.0.0", + env = "HEXSTODY_HOST" + )] + public_host: String, + /// Port to bind the service to + #[clap(long, short, default_value = "8480", env = "HEXSTODY_PORT")] + public_port: u16, + }, + /// Output swagger spec for public API + SwaggerPublic, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let args = Args::parse(); + env_logger::init(); + + match args.subcmd.clone() { + SubCommand::Serve { + public_host, + public_port, + } => loop { + let args = args.clone(); + + info!("Connecting to database"); + let pool = create_db_pool(&args.dbconnect).await?; + info!("Connected"); + + info!("Serving API"); + let public_api_fut = serve_public_api(&public_host, public_port, pool); + match Abortable::new(public_api_fut, abort_api_reg).await { + Ok(mres) => mres?, + Err(Aborted) => { + error!("API thread aborted"); + } + } + + let restart_dt = Duration::from_secs(5); + info!("Adding {:?} delay before restarting logic", restart_dt); + sleep(restart_dt).await; + }, + SubCommand::Swagger => { + let pool = create_db_pool(&args.dbconnect).await?; + let specs = public_api_specs(pool).await?; + let specs_str = serde_json::to_string_pretty(&specs)?; + println!("{}", specs_str); + } + } + Ok(()) +} diff --git a/make-docker.sh b/make-docker.sh new file mode 100755 index 0000000..918ef89 --- /dev/null +++ b/make-docker.sh @@ -0,0 +1 @@ +nix-build ./nix/containers.nix -o docker-image-hexstody.tar.gz "$@" diff --git a/nix/containers.nix b/nix/containers.nix new file mode 100644 index 0000000..522ed65 --- /dev/null +++ b/nix/containers.nix @@ -0,0 +1,40 @@ +{ containerTag ? "latest" +, prefixName ? "" +}: +let + sources = import ./sources.nix; + pkgs = import sources.nixpkgs {}; + hexstody = import ../default.nix; + + baseImage = pkgs.dockerTools.pullImage { + imageName = "debian"; + imageDigest = "sha256:7d8264bf731fec57d807d1918bec0a16550f52a9766f0034b40f55c5b7dc3712"; + sha256 = "sha256-PwMVlEk81ALRwDCSdb9LLdJ1zr6tn4EMxcqtlvxihnE="; + }; + + # As we place all executables in single derivation the derivation takes them + # from it and allows us to make thin containers for each one. + takeOnly = name: path: pkgs.runCommandNoCC "only-${name}" {} '' + mkdir -p $out + cp ${path} $out/${name} + ''; + takeFolder = name: path: innerPath: pkgs.runCommandNoCC "folder-${name}" {} '' + mkdir -p $out/${innerPath} + cp -r ${path}/* $out/${innerPath} + ''; + + mkDockerImage = name: cnts: pkgs.dockerTools.buildImage { + name = "${prefixName}${name}"; + fromImage = baseImage; + tag = containerTag; + contents = cnts; + }; + + hexstody-container = mkDockerImage "hexstody" [ + (takeOnly "hexstody" "${hexstody}/bin/hexstody") + (takeOnly "wait-for-it.sh" "${hexstody.src}/docker/wait-for-it.sh") + ]; +in { inherit + hexstody-container + ; +} diff --git a/nix/overlay.nix b/nix/overlay.nix new file mode 100644 index 0000000..59956a1 --- /dev/null +++ b/nix/overlay.nix @@ -0,0 +1,4 @@ +self: super: +rec { + eclair-tortoise = import ../default.nix; +} diff --git a/nix/pkgs.nix b/nix/pkgs.nix new file mode 100644 index 0000000..0eb8e8e --- /dev/null +++ b/nix/pkgs.nix @@ -0,0 +1,7 @@ +# To update nix-prefetch-git https://github.com/NixOS/nixpkgs +import ((import {}).fetchFromGitHub { + owner = "NixOS"; + repo = "nixpkgs"; + rev = "7ec99ea7cf9616ef4c6e835710202623fcb846e7"; + sha256 = "1cp4sb4v1qzb268h2ky7039lf1gwkrs757q6gv2wd2hs65kvf1q7"; +}) diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 0000000..558c83e --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,50 @@ +{ + "naersk": { + "branch": "master", + "description": "Build rust crates in Nix. No configuration, no code generation, no IFD. Sandbox friendly.", + "homepage": "", + "owner": "nmattia", + "repo": "naersk", + "rev": "b3b099d669fc8b18d361c249091c9fe95d57ebbb", + "sha256": "156fbnr5s2n1xxbbk2z9xa7c5g2z5fdpqmjjs6n9ipbr038n0z3s", + "type": "tarball", + "url": "https://github.com/nmattia/naersk/archive/b3b099d669fc8b18d361c249091c9fe95d57ebbb.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "niv": { + "branch": "master", + "description": "Easy dependency management for Nix projects", + "homepage": "https://github.com/nmattia/niv", + "owner": "nmattia", + "repo": "niv", + "rev": "af958e8057f345ee1aca714c1247ef3ba1c15f5e", + "sha256": "1qjavxabbrsh73yck5dcq8jggvh3r2jkbr6b5nlz5d9yrqm9255n", + "type": "tarball", + "url": "https://github.com/nmattia/niv/archive/af958e8057f345ee1aca714c1247ef3ba1c15f5e.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "release-20.03", + "description": "Nix Packages collection", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6d1a044fc9ff3cc96fca5fa3ba9c158522bbf2a5", + "sha256": "07a3nyrj3pwl017ig0rbn5rbmbf14gl3vqggvkyrdby01726p5fg", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/6d1a044fc9ff3cc96fca5fa3ba9c158522bbf2a5.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-mozilla": { + "branch": "master", + "description": "mozilla related nixpkgs (extends nixos/nixpkgs repo)", + "homepage": "", + "owner": "mozilla", + "repo": "nixpkgs-mozilla", + "rev": "8c007b60731c07dd7a052cce508de3bb1ae849b4", + "sha256": "1zybp62zz0h077zm2zmqs2wcg3whg6jqaah9hcl1gv4x8af4zhs6", + "type": "tarball", + "url": "https://github.com/mozilla/nixpkgs-mozilla/archive/8c007b60731c07dd7a052cce508de3bb1ae849b4.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 0000000..1938409 --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,174 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + if spec ? ref then spec.ref else + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; + in + builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; }; + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else {}; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..253fd8f --- /dev/null +++ b/shell.nix @@ -0,0 +1,21 @@ +with import ./nix/pkgs.nix {}; +let merged-openssl = symlinkJoin { name = "merged-openssl"; paths = [ openssl.out openssl.dev ]; }; +in stdenv.mkDerivation rec { + name = "rust-env"; + env = buildEnv { name = name; paths = buildInputs; }; + + buildInputs = [ + rustup + clang + llvm + llvmPackages.libclang + openssl + cacert + #podman-compose + docker-compose + ]; + shellHook = '' + export LIBCLANG_PATH="${llvmPackages.libclang}/lib" + export OPENSSL_DIR="${merged-openssl}" + ''; +}