Refactor "Add Data" flow to account for ClickHouse-specific forms & YAML

web-common/src/features/connectors/ConnectorEntry.svelte

@@ -2,12 +2,12 @@
   import CaretDownIcon from "../../components/icons/CaretDownIcon.svelte";
   import { Tag } from "../../components/tag";
   import {
-    type V1AnalyzedConnector,
     createRuntimeServiceGetInstance,
+    type V1AnalyzedConnector,
   } from "../../runtime-client";
   import { runtime } from "../../runtime-client/runtime-store";
   import type { ConnectorExplorerStore } from "./connector-explorer-store";
-  import { connectorIconMapping } from "./connector-icon-mapping";
+  import { symbolIconMapping } from "./connector-icon-mapping";
   import DatabaseExplorer from "./olap/DatabaseExplorer.svelte";
 
   export let connector: V1AnalyzedConnector;
@@ -44,7 +44,7 @@
         <span class="flex-none">
           {#if connector.driver?.name}
             <svelte:component
-              this={connectorIconMapping[connector.driver.name]}
+              this={symbolIconMapping[connector.driver.name]}
               size="16px"
             />
           {/if}

web-common/src/features/connectors/clickhouse/source-templates.ts

@@ -0,0 +1,36 @@
+import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client";
+
+export function compileClickhouseSourceConnectorFile(
+  connector: V1ConnectorDriver,
+  formValues: Record<string, unknown>,
+) {
+  switch (connector.name) {
+    case "azure":
+      return `sql: SELECT * FROM azure('${formValues.path as string}');
+
+output:
+  materialize: true`;
+    case "gcs":
+      return `sql: SELECT * FROM gcs('${formValues.path as string}', '{{ .env.connector.gcs.hmac_key }}', '{{ .env.connector.gcs.hmac_secret }}');
+
+output:
+  materialize: true`;
+    case "s3":
+      return `sql: SELECT * FROM s3('${formValues.path as string}', '{{ .env.connector.s3.aws_access_key_id }}', '{{ .env.connector.s3.aws_secret_access_key }}');
+
+output:
+  materialize: true`;
+    case "postgres":
+      return `sql: SELECT * FROM postgresql('${formValues.host as string}:${formValues.port as string}', '${formValues.database as string}', '${formValues.table as string}', '${formValues.user as string}', '{{ .env.connector.postgres.password }}'
+
+output:
+  materialize: true`;
+    case "mysql":
+      return `sql: SELECT * FROM mysql('${formValues.host as string}:${formValues.port as string}', '${formValues.database as string}', '${formValues.user as string}', '{{ .env.connector.mysql.password }}');
+
+output:
+  materialize: true`;
+    default:
+      throw new Error(`Unsupported connector: ${connector.name}`);
+  }
+}

web-common/src/features/connectors/connector-availability.ts

@@ -0,0 +1,27 @@
+export const DUCKDB_SOURCE_CONNECTORS = [
+  "duckdb",
+  "motherduck",
+  "postgresql",
+  "gcs",
+  "bigquery",
+  "snowflake",
+  "s3",
+  "athena",
+  "redshift",
+  "mysql",
+  "sqlite",
+  "azure",
+  "salesforce",
+  "local_file",
+  "https",
+];
+
+export const CLICKHOUSE_SOURCE_CONNECTORS = [
+  "postgres",
+  "mysql",
+  "gcs",
+  "s3",
+  "azure",
+];
+
+export const OLAP_CONNECTORS = ["clickhouse", "druid", "pinot"];

web-common/src/features/connectors/connector-icon-mapping.ts

@@ -1,3 +1,21 @@
+import AmazonAthena from "@rilldata/web-common/components/icons/connectors/AmazonAthena.svelte";
+import AmazonRedshift from "@rilldata/web-common/components/icons/connectors/AmazonRedshift.svelte";
+import AmazonS3 from "@rilldata/web-common/components/icons/connectors/AmazonS3.svelte";
+import ApacheDruid from "@rilldata/web-common/components/icons/connectors/ApacheDruid.svelte";
+import ApachePinot from "@rilldata/web-common/components/icons/connectors/ApachePinot.svelte";
+import ClickHouse from "@rilldata/web-common/components/icons/connectors/ClickHouse.svelte";
+import DuckDb from "@rilldata/web-common/components/icons/connectors/DuckDB.svelte";
+import GoogleBigQuery from "@rilldata/web-common/components/icons/connectors/GoogleBigQuery.svelte";
+import GoogleCloudStorage from "@rilldata/web-common/components/icons/connectors/GoogleCloudStorage.svelte";
+import Https from "@rilldata/web-common/components/icons/connectors/HTTPS.svelte";
+import LocalFile from "@rilldata/web-common/components/icons/connectors/LocalFile.svelte";
+import MicrosoftAzureBlobStorage from "@rilldata/web-common/components/icons/connectors/MicrosoftAzureBlobStorage.svelte";
+import MotherDuck from "@rilldata/web-common/components/icons/connectors/MotherDuck.svelte";
+import MySql from "@rilldata/web-common/components/icons/connectors/MySQL.svelte";
+import Postgres from "@rilldata/web-common/components/icons/connectors/Postgres.svelte";
+import SqLite from "@rilldata/web-common/components/icons/connectors/SQLite.svelte";
+import Salesforce from "@rilldata/web-common/components/icons/connectors/Salesforce.svelte";
+import Snowflake from "@rilldata/web-common/components/icons/connectors/Snowflake.svelte";
 import { SnowflakeIcon } from "lucide-svelte";
 import AmazonS3Icon from "../../components/icons/connectors/AmazonS3Icon.svelte";
 import ApacheDruidIcon from "../../components/icons/connectors/ApacheDruidIcon.svelte";
@@ -6,18 +24,35 @@ import ClickHouseIcon from "../../components/icons/connectors/ClickHouseIcon.sve
 import DuckDbIcon from "../../components/icons/connectors/DuckDBIcon.svelte";
 import GoogleBigQueryIcon from "../../components/icons/connectors/GoogleBigQueryIcon.svelte";
 
-export const connectorIconMapping = {
-  // TODO: athena
-  // TODO: azure
+// These are symbols only (no text)
+export const symbolIconMapping = {
   bigquery: GoogleBigQueryIcon,
   clickhouse: ClickHouseIcon,
   druid: ApacheDruidIcon,
   duckdb: DuckDbIcon,
-  // TODO: gcs
-  // TODO: mysql
   pinot: ApachePinotIcon,
-  // TODO: postgres
-  // TODO: redshift
   s3: AmazonS3Icon,
   snowflake: SnowflakeIcon,
 };
+
+// These are full logos (w/ text)
+export const logoIconMapping = {
+  athena: AmazonAthena,
+  azure: MicrosoftAzureBlobStorage,
+  bigquery: GoogleBigQuery,
+  clickhouse: ClickHouse,
+  druid: ApacheDruid,
+  duckdb: DuckDb,
+  gcs: GoogleCloudStorage,
+  https: Https,
+  local_file: LocalFile,
+  motherduck: MotherDuck,
+  mysql: MySql,
+  pinot: ApachePinot,
+  postgres: Postgres,
+  redshift: AmazonRedshift,
+  s3: AmazonS3,
+  salesforce: Salesforce,
+  snowflake: Snowflake,
+  sqlite: SqLite,
+};

web-common/src/features/connectors/olap/olap-config.ts

@@ -1,3 +1,7 @@
+export const OLAP_CONNECTOR_NAMES = ["duckdb", "clickhouse", "druid"];
+
+export type OlapDriver = (typeof OLAP_CONNECTOR_NAMES)[number];
+
 export const OLAP_DRIVERS_WITHOUT_MODELING = ["clickhouse", "druid", "pinot"];
 
 export function makeFullyQualifiedTableName(

web-common/src/features/connectors/olap/selectors.ts

@@ -6,10 +6,78 @@ import {
   createConnectorServiceOLAPListTables,
   createRuntimeServiceAnalyzeConnectors,
   createRuntimeServiceGetInstance,
+  createRuntimeServiceListConnectorDrivers,
 } from "../../../runtime-client";
 import { featureFlags } from "../../feature-flags";
+import {
+  CLICKHOUSE_SOURCE_CONNECTORS,
+  DUCKDB_SOURCE_CONNECTORS,
+} from "../connector-availability";
 import { OLAP_DRIVERS_WITHOUT_MODELING } from "./olap-config";
 
+export function useCurrentOlapConnector(instanceId: string) {
+  return createRuntimeServiceGetInstance(
+    instanceId,
+    { sensitive: true },
+    {
+      query: {
+        select: (data) => {
+          const {
+            instance: {
+              olapConnector: olapConnectorName = "",
+              connectors = [],
+              projectConnectors = [],
+            } = {},
+          } = data || {};
+          const olapConnector = [...connectors, ...projectConnectors].find(
+            (connector) => connector.name === olapConnectorName,
+          );
+          return olapConnector;
+        },
+      },
+    },
+  );
+}
+
+export function useSourceConnectorsForCurrentOlapConnector(instanceId: string) {
+  return derived(
+    [
+      createRuntimeServiceListConnectorDrivers(),
+      useCurrentOlapConnector(instanceId),
+    ],
+    ([connectors, olapConnector]) => {
+      const allConnectorDrivers = connectors.data?.connectors ?? [];
+      const olapConnectorType = olapConnector.data?.type;
+
+      if (!allConnectorDrivers || !olapConnectorType) {
+        return [];
+      }
+
+      const sourceConnectorNames = (
+        olapConnectorType === "clickhouse"
+          ? CLICKHOUSE_SOURCE_CONNECTORS
+          : olapConnectorType === "duckdb"
+            ? DUCKDB_SOURCE_CONNECTORS
+            : []
+      ) as string[];
+
+      const sourceConnectors = allConnectorDrivers
+        .filter((a) => {
+          return a.name && sourceConnectorNames.includes(a.name);
+        })
+        .sort(
+          // CAST SAFETY: we have filtered out any connectors that
+          // don't have a `name` in the previous filter
+          (a, b) =>
+            sourceConnectorNames.indexOf(a.name as string) -
+            sourceConnectorNames.indexOf(b.name as string),
+        );
+
+      return sourceConnectors;
+    },
+  );
+}
+
 export function useIsModelingSupportedForOlapDriver(
   instanceId: string,
   driver: string,

web-common/src/features/entity-management/entity-mappers.ts

@@ -1,44 +1,6 @@
 import { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors";
 import { EntityType } from "@rilldata/web-common/features/entity-management/types";
 
-export function getFilePathFromPagePath(path: string): string {
-  const pathSplits = path.split("/");
-  const entityType = pathSplits[1];
-  const entityName = pathSplits[2];
-
-  switch (entityType) {
-    case "source":
-      return `/files/sources/${entityName}.yaml`;
-    case "model":
-      return `/files/models/${entityName}.sql`;
-    case "dashboard":
-      return `/files/dashboards/${entityName}.yaml`;
-    default:
-      throw new Error("type must be either 'source', 'model', or 'dashboard'");
-  }
-}
-
-export function getFilePathFromNameAndType(
-  name: string,
-  type: EntityType,
-): string {
-  switch (type) {
-    case EntityType.Connector:
-      return `/connectors/${name}.yaml`;
-    case EntityType.Table:
-      return `/sources/${name}.yaml`;
-    case EntityType.Model:
-      return `/models/${name}.sql`;
-    case EntityType.MetricsDefinition:
-      return `/dashboards/${name}.yaml`;
-    case EntityType.Chart:
-      return `/charts/${name}.yaml`;
-    case EntityType.Canvas:
-      return `/canvas/${name}.yaml`;
-    default:
-      throw new Error("Unrecognized EntityType");
-  }
-}
 // Temporary solution for the issue with leading `/` for these files.
 // TODO: find a solution that works across backend and frontend
 export function getFileAPIPathFromNameAndType(
@@ -48,6 +10,7 @@ export function getFileAPIPathFromNameAndType(
   switch (type) {
     case EntityType.Connector:
       return `connectors/${name}.yaml`;
+    case EntityType.Source:
     case EntityType.Table:
       return `sources/${name}.yaml`;
     case EntityType.Model:
@@ -70,43 +33,6 @@ export function getNameFromFile(fileName: string): string {
   return extensionSplits[0];
 }
 
-export function getRouteFromName(name: string, type: EntityType): string {
-  if (!name) return "/";
-  switch (type) {
-    case EntityType.Table:
-      return `/files/source/${name}`;
-    case EntityType.Model:
-      return `/files/model/${name}`;
-    case EntityType.MetricsDefinition:
-      return `/files/dashboard/${name}`;
-    case EntityType.Chart:
-      return `/files/chart/${name}`;
-    case EntityType.Canvas:
-      return `/files/canvas/${name}`;
-    default:
-      throw new Error("Unrecognized EntityType");
-  }
-}
-
-export function getLabel(entityType: EntityType) {
-  switch (entityType) {
-    case EntityType.Table:
-      return "source";
-    case EntityType.Model:
-      return "model";
-    case EntityType.MetricsDefinition:
-      return "dashboard";
-    case EntityType.Chart:
-      return "chart";
-    case EntityType.Canvas:
-      return "canvas";
-    case EntityType.Unknown:
-      return "";
-    default:
-      throw new Error("Unrecognized EntityType");
-  }
-}
-
 // Remove a leading slash, if it exists
 // In certain backend APIs where path is part of the url.
 // Leading slash leads to issues where the end point redirects to one without the slash.

web-common/src/features/entity-management/types.ts

@@ -1,7 +1,8 @@
 export enum EntityType {
   Connector = "Connector",
-  Table = "Table",
+  Source = "Source",
   Model = "Model",
+  Table = "Table",
   Application = "Application",
   MetricsDefinition = "MetricsDefinition",
   MetricsExplorer = "MetricsExplorer",