feat(file imports): adds an experimental IFC parser (#3525)

* first pass of CLI ifc converter

* some updates

* closer

* yarn works

* can execute ifc?

* change exe

* remove extra venv needs

* invocation works

* fixed dockerfile and url

* refactor(fileimport): temp results path should not be hardcoded in parsers

* update importer to output stuff

* fix up argments

* remove dead code

* adjust dockerfile to have tini and workdir better

* fix node to a specific version

* Add shell statement and pin yarn version

* add ifc converter c# to ignore

* merge fix

* move ifc c#

* fix the api usage

* update the importer to new SDK

* Adds a feature flag `FF_FILEIMPORT_IFC_DOTNET_ENABLED` for enabling .Net IFC parser

* move directories

* put back ifc js

* use FF and reversions

* needs token too

* fix docker?

* one last copy fix

* adjust prettier ignore

* change to enable

* fix helm chart nesting

* Amend healthcheck node binary path

* Add FF_FILEIMPORT_IFC_DOTNET_ENABLED to feature flag parser

* Allow app to write to /.config directory

* fix: volume name has to be lower case

* update ifc importing

---------

Co-authored-by: Iain Sproat <[email protected]>
Co-authored-by: root <root@Clynelish>

.gitignore

@@ -72,6 +72,9 @@ postgres-data/
 redis-data/
 
 .tshy-build
+obj/
+bin/
+
 
 # Server
 multiregion.json

.prettierignore

@@ -38,4 +38,8 @@ venv
 
 storybook-static
 .tshy
-.tshy-build
+.tshy-build
+
+packages/fileimport-service/ifc-dotnet/
+packages/fileimport-service/stl/
+packages/fileimport-service/obj/

packages/fileimport-service/Dockerfile

@@ -1,97 +1,75 @@
 ARG NODE_ENV=production
 
-FROM node:18-bookworm-slim@sha256:408f8cbbb7b33a5bb94bdb8862795a94d2b64c2d516856824fd86c4a5594a443 as build-stage
+FROM mcr.microsoft.com/dotnet/sdk:8.0-noble AS dotnet-build-stage
+WORKDIR /app
+COPY packages/fileimport-service/ifc-dotnet .
+RUN dotnet publish ifc-converter.csproj -c Release -o output/
 
-ARG NODE_ENV
-ENV NODE_ENV=${NODE_ENV}
+
+FROM mcr.microsoft.com/dotnet/runtime:8.0-noble AS runtime
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 WORKDIR /speckle-server
 
 # install tini
 ARG TINI_VERSION=v0.19.0
 ENV TINI_VERSION=${TINI_VERSION}
-ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini ./tini
-RUN chmod +x ./tini
-
-# install wait
-ARG WAIT_VERSION=2.8.0
-ENV WAIT_VERSION=${WAIT_VERSION}
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/${WAIT_VERSION}/wait ./wait
-RUN chmod +x ./wait
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
+RUN chmod +x /usr/bin/tini
+
+RUN apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+     --no-install-recommends \
+     curl=8.5.0-2ubuntu10.5 \
+    && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+     --no-install-recommends \
+     nodejs=18.20.5-1nodesource1 \
+    && npm install -g [email protected] \
+    && corepack enable \
+    && DEBIAN_FRONTEND=noninteractive apt-get remove curl -y \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # download yarn dependencies for building shared libraries
 COPY .yarnrc.yml .
 COPY .yarn ./.yarn
 COPY package.json yarn.lock ./
 
-COPY packages/frontend-2/type-augmentations/stubs ./packages/frontend-2/type-augmentations/stubs/
-COPY packages/shared/package.json ./packages/shared/
-COPY packages/fileimport-service/package.json ./packages/fileimport-service/
+COPY packages/frontend-2/type-augmentations/stubs packages/frontend-2/type-augmentations/stubs/
+COPY packages/shared/package.json packages/shared/
+COPY packages/fileimport-service/package.json packages/fileimport-service/
 
 RUN yarn workspaces focus --all
 
 # build shared libraries
-COPY packages/shared ./packages/shared/
-COPY packages/fileimport-service ./packages/fileimport-service/
+COPY packages/shared packages/shared/
+COPY packages/fileimport-service packages/fileimport-service/
 RUN yarn workspaces foreach -W run build
 
 # Install python virtual env and python dependencies
-RUN apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install -y \
-  --no-install-recommends \
-  python3-venv=3.11.2-1+b1 \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/* \
-  && python3 -m venv /venv
+RUN apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+     --no-install-recommends \
+     python3.12=3.12.3-1ubuntu0.3 \
+     python3-pip=24.0+dfsg-1ubuntu1.1 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY packages/fileimport-service/requirements.txt /speckle-server/
-RUN /venv/bin/pip install --disable-pip-version-check --no-cache-dir --requirement /speckle-server/requirements.txt
-
-FROM node:18-bookworm-slim@sha256:408f8cbbb7b33a5bb94bdb8862795a94d2b64c2d516856824fd86c4a5594a443 as dependency-stage
-# installing just the production dependencies
-# separate stage to avoid including development dependencies
-ARG NODE_ENV
-ENV NODE_ENV=${NODE_ENV}
-
-WORKDIR /speckle-server
-COPY .yarnrc.yml .
-COPY .yarn ./.yarn
-COPY package.json yarn.lock ./
-
-COPY packages/frontend-2/type-augmentations/stubs ./packages/frontend-2/type-augmentations/stubs/
-COPY packages/shared/package.json ./packages/shared/
-COPY packages/fileimport-service/package.json ./packages/fileimport-service/
-
-WORKDIR /speckle-server/packages/fileimport-service
-RUN yarn workspaces focus --production
-
-FROM gcr.io/distroless/python3-debian12:nonroot@sha256:14c62b8925d3bb30319de2f346bde203fe18103a68898284a62db9d4aa54c794 as python-image
-
-FROM gcr.io/distroless/nodejs18-debian12:nonroot@sha256:afdea027580f7afcaf1f316b2b3806690c297cb3ce6ddc5cf6a15804dc1c790f as distributable-stage
+RUN pip install --break-system-packages --disable-pip-version-check --no-cache-dir --requirement /speckle-server/requirements.txt
 
 ARG NODE_ENV
 ENV NODE_ENV=${NODE_ENV}
-ARG NODE_BINARY_PATH=/nodejs/bin/node
+ARG NODE_BINARY_PATH=/usr/bin/node
 ENV NODE_BINARY_PATH=${NODE_BINARY_PATH}
-ARG PYTHON_BINARY_PATH=/venv/bin/python3
+ARG PYTHON_BINARY_PATH=/usr/bin/python3
 ENV PYTHON_BINARY_PATH=${PYTHON_BINARY_PATH}
+ARG DOTNET_BINARY_PATH=/usr/bin/dotnet
+ENV DOTNET_BINARY_PATH=${DOTNET_BINARY_PATH}
 
-WORKDIR /speckle-server
-
-COPY --from=python-image / /
-COPY --from=build-stage /speckle-server/tini /usr/bin/tini
-COPY --from=build-stage /speckle-server/wait /usr/bin/wait
-COPY --from=build-stage /speckle-server/packages/shared ./packages/shared
-COPY --from=build-stage /speckle-server/packages/fileimport-service ./packages/fileimport-service
-COPY --from=build-stage /venv /venv
-COPY --from=dependency-stage /speckle-server/node_modules ./node_modules
+COPY --from=dotnet-build-stage /app/output packages/fileimport-service/ifc-dotnet
 
 WORKDIR /speckle-server/packages/fileimport-service
 
-# Prefixing PATH with our virtual environment should seek required binaries
-# from virtual environment first.
-# Unsetting python home
-ENV PATH=/venv/bin:${PATH} \
-    PYTHONHOME=
-
-ENTRYPOINT [ "tini", "--", "/nodejs/bin/node", "--no-experimental-fetch", "src/daemon.js"]
+ENTRYPOINT [ "tini", "--", "node", "--no-experimental-fetch", "src/daemon.js"]

packages/fileimport-service/eslint.config.mjs

@@ -5,6 +5,9 @@ import { baseConfigs, globals } from '../../eslint.config.mjs'
  */
 const configs = [
   ...baseConfigs,
+  {
+    ignores: ['**/ifc/**', '**/obj/**', '**/stl/**']
+  },
   {
     languageOptions: {
       globals: {

packages/fileimport-service/ifc-dotnet/.config/dotnet-tools.json

@@ -0,0 +1,12 @@
+{
+  "version": 1,
+  "isRoot": true,
+  "tools": {
+    "csharpier": {
+      "version": "0.30.1",
+      "commands": [
+        "dotnet-csharpier"
+      ]
+    }
+  }
+}

packages/fileimport-service/ifc-dotnet/ConsoleProgress.cs

@@ -0,0 +1,33 @@
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Converter;
+
+public class ConsoleProgress : IProgress<ProgressArgs>
+{
+  private readonly TimeSpan DEBOUNCE = TimeSpan.FromSeconds(1);
+  private DateTime _lastTime = DateTime.UtcNow;
+
+  private long _totalBytes;
+
+  public void Report(ProgressArgs value)
+  {
+    if (value.ProgressEvent == ProgressEvent.DownloadBytes)
+    {
+      Interlocked.Add(ref _totalBytes, value.Count);
+    }
+    var now = DateTime.UtcNow;
+    if (now - _lastTime >= DEBOUNCE)
+    {
+      if (value.ProgressEvent == ProgressEvent.DownloadBytes)
+      {
+        Console.WriteLine(value.ProgressEvent + " t " + _totalBytes);
+      }
+      else
+      {
+        Console.WriteLine(value.ProgressEvent + " c " + value.Count + " t " + value.Total);
+      }
+
+      _lastTime = now;
+    }
+  }
+}

packages/fileimport-service/ifc-dotnet/Program.cs

@@ -0,0 +1,47 @@
+using System.CommandLine;
+using System.Text.Json;
+using Speckle.Sdk.Common;
+using Speckle.WebIfc.Importer;
+
+var filePathArgument = new Argument<string>(name: "filePath");
+var outputPathArgument = new Argument<string>("outputPath");
+var streamIdArgument = new Argument<string>("streamId");
+var commitMessageArgument = new Argument<string>("commitMessage");
+var modelIdArgument = new Argument<string>("modelId");
+var regionNameArgument = new Argument<string>("regionName");
+
+var rootCommand = new RootCommand
+{
+  filePathArgument,
+  outputPathArgument,
+  streamIdArgument,
+  commitMessageArgument,
+  modelIdArgument,
+  regionNameArgument,
+};
+rootCommand.SetHandler(
+  async (filePath, outputPath, streamId, commitMessage, modelId, _) =>
+  {
+    try
+    {
+      var token = Environment.GetEnvironmentVariable("USER_TOKEN").NotNull("USER_TOKEN is missing");
+      var url = Environment.GetEnvironmentVariable("SPECKLE_SERVER_URL") ?? "http://127.0.0.1:3000";
+      var commitId = await Import.Ifc(url, filePath, streamId, modelId, commitMessage, token);
+      File.WriteAllText(outputPath, JsonSerializer.Serialize(new { success = true, commitId }));
+    }
+    catch (Exception e)
+    {
+      File.WriteAllText(
+        outputPath,
+        JsonSerializer.Serialize(new { success = false, error = e.ToString() })
+      );
+    }
+  },
+  filePathArgument,
+  outputPathArgument,
+  streamIdArgument,
+  commitMessageArgument,
+  modelIdArgument,
+  regionNameArgument
+);
+await rootCommand.InvokeAsync(args);

packages/fileimport-service/ifc-dotnet/ifc-converter.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <RootNamespace>Speckle.Converter</RootNamespace>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
+      <PackageReference Include="Speckle.WebIfc.Importer" Version="0.0.5" />
+      <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+    </ItemGroup>
+
+</Project>

packages/fileimport-service/ifc-dotnet/ifc-converter.sln

@@ -0,0 +1,16 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ifc-converter", "ifc-converter.csproj", "{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal