diff --git a/api/project/__init__.py b/api/project/__init__.py index 46a7c0111..3e1d45f65 100644 --- a/api/project/__init__.py +++ b/api/project/__init__.py @@ -1062,6 +1062,96 @@ def ready(): ) +@app.route( + f"{('/' + app.config['MALCOLM_API_PREFIX']) if app.config['MALCOLM_API_PREFIX'] else ''}/dashboard-export/", + methods=['GET', 'POST'], +) +def dashboard_export(dashid): + """Uses the opensearch dashboards API to export a dashboard. Also handles the _REPLACER strings + as described in "Adding new visualizations and dashboards" at + https://idaholab.github.io/Malcolm/docs/contributing-dashboards.html#DashboardsNewViz + + Parameters + ---------- + dashid : string + the ID of the dashboard to export + request : Request + Uses 'replace' from requests arguments, true (default) or false; indicates whether or not to do + MALCOLM_NETWORK_INDEX_PATTERN_REPLACER, MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER, + MALCOLM_OTHER_INDEX_PATTERN_REPLACER + + Returns + ------- + content + The JSON of the exported dashboard + """ + + args = get_request_arguments(request) + try: + # call the API to get the dashboard JSON + response = requests.get( + f"{dashboardsUrl}/api/{'kibana' if (databaseMode == malcolm_utils.DatabaseMode.ElasticsearchRemote) else 'opensearch-dashboards'}/dashboards/export", + params={ + 'dashboard': dashid, + }, + auth=opensearchReqHttpAuth, + verify=opensearchSslVerify, + ) + response.raise_for_status() + + if doReplacers := malcolm_utils.str2bool(args.get('replace', 'true')): + # replace references to index pattern names with the _REPLACER strings, which will allow other Malcolm + # instances that use different index pattern names to import them and substitute their own names + replacements = { + app.config['MALCOLM_NETWORK_INDEX_PATTERN']: 'MALCOLM_NETWORK_INDEX_PATTERN_REPLACER', + app.config['MALCOLM_NETWORK_INDEX_TIME_FIELD']: 'MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER', + app.config['MALCOLM_OTHER_INDEX_PATTERN']: 'MALCOLM_OTHER_INDEX_PATTERN_REPLACER', + } + pattern = re.compile('|'.join(re.escape(key) for key in replacements)) + responseText = pattern.sub(lambda match: replacements[match.group(0)], response.text) + else: + # ... or just return it as-is + responseText = response.text + + # remove index pattern definition from exported dashboard as they get created programatically + # on Malcolm startup and we don't want them to come in with imported dashboards + if responseParsed := malcolm_utils.LoadStrIfJson(responseText): + if 'objects' in responseParsed and isinstance(responseParsed['objects'], list): + responseParsed['objects'] = [ + o + for o in responseParsed['objects'] + if not ( + (o.get("type") == "index-pattern") + and ( + o.get("id") + in [ + ( + "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + if doReplacers + else app.config['MALCOLM_NETWORK_INDEX_PATTERN'] + ), + ( + "MALCOLM_OTHER_INDEX_PATTERN_REPLACER" + if doReplacers + else app.config['MALCOLM_OTHER_INDEX_PATTERN'] + ), + ] + ) + ) + ] + return jsonify(responseParsed) + + else: + # what we got back from the API wasn't valid JSON, so sad + return jsonify(error=f'Could not process export response for {dashid}') + + except Exception as e: + errStr = f"{type(e).__name__}: {str(e)} exporting OpenSearch Dashboard {dashid}" + if debugApi: + print(errStr) + return jsonify(error=errStr) + + @app.route( f"{('/' + app.config['MALCOLM_API_PREFIX']) if app.config['MALCOLM_API_PREFIX'] else ''}/ingest-stats", methods=['GET'], diff --git a/docs/api-dashboard-export.md b/docs/api-dashboard-export.md new file mode 100644 index 000000000..6a4450c25 --- /dev/null +++ b/docs/api-dashboard-export.md @@ -0,0 +1,35 @@ +# Dashboard Export + +`GET` or `POST` - /mapi/dashboard-export/`` + +Uses the [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/) or [Elastic Kibana](https://www.elastic.co/guide/en/kibana/current/dashboard-api-export.html) API to export the JSON document representing a dashboard (identified by `dashid`). If the query parameter `replace` is not set to `false`, this API will also perform some modifications on the dashboard as described in the [**Adding new visualizations and dashboards**](contributing-dashboards.md#DashboardsNewViz) section of the [contributor guide](contributing-guide.md#Contributing). + +Parameters: + +* `dashid` (URL parameter) - the [ID of the dashboard]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/dashboards/dashboards/) to be exported (e.g., `0ad3d7c2-3441-485e-9dfe-dbb22e84e576`) +* `replace` (query parameter) - whether or not to perform the index pattern name replacements as described above (default: `true`) + +Example (response truncated for brevity's sake): + +``` +/mapi/dashboard-export/0ad3d7c2-3441-485e-9dfe-dbb22e84e576 +``` + +```json +{ + + "version": "1.3.1", + "objects": [ + { + "id": "0ad3d7c2-3441-485e-9dfe-dbb22e84e576", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-04-29T15:49:16.000Z", + "version": "WzEzNjIsMV0=", + "attributes": { + "title": "Overview" +… +} +``` \ No newline at end of file diff --git a/docs/api.md b/docs/api.md index b467631d4..cf6bb475a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,5 +1,6 @@ # API +* [Dashboard Export](api-dashboard-export.md) * [Document Ingest Statistics](api-ingest-stats.md) * [Document Lookup](api-document-lookup.md) * [Event Logging](api-event-logging.md) diff --git a/docs/contributing-dashboards.md b/docs/contributing-dashboards.md index 313e89fee..f37cb0499 100644 --- a/docs/contributing-dashboards.md +++ b/docs/contributing-dashboards.md @@ -4,36 +4,62 @@ ## Adding new visualizations and dashboards -Visualizations and dashboards can be [easily created](dashboards.md#BuildDashboard) in OpenSearch Dashboards using its drag-and-drop WYSIWIG tools. Assuming users have created a new dashboard to package with Malcolm, the dashboard and its visualization components can be exported using the following steps: +Visualizations and dashboards can be [easily created](dashboards.md#BuildDashboard) in OpenSearch Dashboards using its drag-and-drop WYSIWIG tools. Assuming users have created a new dashboard to package with Malcolm, the dashboard and its visualization components can be exported either of two ways. + +The easier (and preferred) method is to use the [dashboard export API](api-dashboard-export.md), as it handles the replacers (described below in the more complicated method) automatically.: 1. Identify the ID of the dashboard (found in the URL: e.g., for `/dashboards/app/dashboards#/view/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` the ID would be `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`) -1. Export the dashboard with that ID and save it in the `./dashboards./dashboards/` directory with the following command: - ``` - export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \ - docker compose exec dashboards curl -XGET \ - "http://localhost:5601/dashboards/api/opensearch-dashboards/dashboards/export?dashboard=$DASHID" > \ - ./dashboards/dashboards/$DASHID.json - ``` -1. It is preferrable for Malcolm to dynamically create the `arkime_sessions3-*` index template rather than including it in imported dashboards, so edit the `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` that was generated, carefully locating and removing the section with the `id` of `arkime_sessions3-*` and the `type` of `index-pattern` (including the comma preceding it): - ``` - , - { - "id": "arkime_sessions3-*", - "type": "index-pattern", - "namespaces": [ - "default" - ], - "updated_at": "2021-12-13T18:21:42.973Z", - "version": "Wzk3MSwxXQ==", - … - "references": [], - "migrationVersion": { - "index-pattern": "7.6.0" - } - } - ``` -1. In your text editor, perform a global-search and replace, replacing the string `arkime_sessions3-*` with `MALCOLM_NETWORK_INDEX_PATTERN_REPLACER` and `malcolm_beats_*` with `MALCOLM_OTHER_INDEX_PATTERN_REPLACER`. These replacers are used to [allow customizing indexes for logs written to OpenSearch or Elasticsearch](https://github.com/idaholab/Malcolm/issues/313). -1. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up. + +2. Using a web browser, enter the URL **https://localhost/mapi/dashboard-export/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx**, replacing `localhost` with the IP address or hostname of your Malcolm instance and the placeholder dashboard ID with the ID you identified in the previous step. Save the raw JSON document returned as `./dashboards/dashboards/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` (using the actual ID) under your Malcolm directory. + +**OR** + +2. Using the command line, export the dashboard with that ID and save it in the `./dashboards/dashboards/` directory with the following command: + +``` +export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \ + docker compose exec api curl -sSL -XGET "http://localhost:5000/mapi/dashboard-export/$DASHID" > \ + ./dashboards/dashboards/$DASHID.json +``` + +3. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up. + + +The manual, more complicated way, consists of the following steps: + +1. Identify the ID of the dashboard (found in the URL: e.g., for `/dashboards/app/dashboards#/view/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` the ID would be `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`) + +2. Using the command line, export the dashboard with that ID and save it in the `./dashboards/dashboards/` directory with the following command: + +``` +export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \ + docker compose exec dashboards curl -XGET \ + "http://localhost:5601/dashboards/api/opensearch-dashboards/dashboards/export?dashboard=$DASHID" > \ + ./dashboards/dashboards/$DASHID.json +``` + +3. It is preferrable for Malcolm to dynamically create the `arkime_sessions3-*` index template rather than including it in imported dashboards, so edit the `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` that was generated, carefully locating and removing the section with the `id` of `arkime_sessions3-*` and the `type` of `index-pattern` (including the comma preceding it): + +``` + , + { + "id": "arkime_sessions3-*", + "type": "index-pattern", + "namespaces": [ + "default" + ], + "updated_at": "2021-12-13T18:21:42.973Z", + "version": "Wzk3MSwxXQ==", + … + "references": [], + "migrationVersion": { + "index-pattern": "7.6.0" + } + } +``` + +4. In your text editor, perform a global-search and replace, replacing the string `arkime_sessions3-*` with `MALCOLM_NETWORK_INDEX_PATTERN_REPLACER` and `malcolm_beats_*` with `MALCOLM_OTHER_INDEX_PATTERN_REPLACER`. These replacers are used to [allow customizing indexes for logs written to OpenSearch or Elasticsearch](https://github.com/idaholab/Malcolm/issues/313). +5. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up. ## OpenSearch Dashboards plugins