From 0c773efd406d8bd7b45e0ed697d288bcca154cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Fri, 11 Oct 2024 21:19:53 +0100 Subject: [PATCH 1/5] remove helm from code package (#1071) --- docs/README.md | 3 +- helm/README.md | 75 ----------- helm/ballista/.gitignore | 3 - helm/ballista/.helmignore | 37 ------ helm/ballista/Chart.lock | 6 - helm/ballista/Chart.yaml | 45 ------- helm/ballista/templates/NOTES.txt | 40 ------ helm/ballista/templates/_helpers.tpl | 77 ------------ helm/ballista/templates/executor.yaml | 107 ---------------- helm/ballista/templates/hpa.yaml | 46 ------- helm/ballista/templates/ingress.yaml | 79 ------------ helm/ballista/templates/scheduler.yaml | 106 ---------------- helm/ballista/templates/serviceaccount.yaml | 30 ----- .../templates/tests/test-connection.yaml | 33 ----- helm/ballista/values.yaml | 116 ------------------ 15 files changed, 1 insertion(+), 802 deletions(-) delete mode 100644 helm/README.md delete mode 100644 helm/ballista/.gitignore delete mode 100644 helm/ballista/.helmignore delete mode 100644 helm/ballista/Chart.lock delete mode 100644 helm/ballista/Chart.yaml delete mode 100644 helm/ballista/templates/NOTES.txt delete mode 100644 helm/ballista/templates/_helpers.tpl delete mode 100644 helm/ballista/templates/executor.yaml delete mode 100644 helm/ballista/templates/hpa.yaml delete mode 100644 helm/ballista/templates/ingress.yaml delete mode 100644 helm/ballista/templates/scheduler.yaml delete mode 100644 helm/ballista/templates/serviceaccount.yaml delete mode 100644 helm/ballista/templates/tests/test-connection.yaml delete mode 100644 helm/ballista/values.yaml diff --git a/docs/README.md b/docs/README.md index e484592c3..a89ad1c21 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,7 +21,7 @@ ## User Documentation -Documentation for the current published release can be found at https://datafusion.apache.org/ballista and the source +Documentation for the current published release can be found at and the source content is located [here](source/user-guide/introduction.md). ## Developer Documentation @@ -37,7 +37,6 @@ inside a Python virtualenv. - Python - `pip install -r requirements.txt` -- DataFusion python package. You can install the latest version by running `maturin develop` inside `../python` directory. ## Build diff --git a/helm/README.md b/helm/README.md deleted file mode 100644 index c24749d5e..000000000 --- a/helm/README.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# Ballista Helm Chart - -Helm is a convenient way to install applications into Kubernetes. It can work against any Kubeneretes environement, -including all the major cloud providers. -For the purposes of this documentation, we will use Kubernetes-in-Docker (kind) to install locally. - -## Prerequisites - -```shell -sudo apt install docker.io docker-compose -sudo snap install go kubectl helm --classic -go install sigs.k8s.io/kind@v0.16.0 -~/go/bin/kind create cluster -``` - -## Build - -```shell -dev/build-ballista-docker.sh -``` - -## Deploy - -```shell -# See https://iximiuz.com/en/posts/kubernetes-kind-load-docker-image/ -# https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster -~/go/bin/kind load docker-image ballista-scheduler:latest -~/go/bin/kind load docker-image ballista-executor:latest - -cd helm/ballista -helm repo add bitnami https://charts.bitnami.com/bitnami -helm dep update -helm dep build -helm install ballista . -``` - -## Access Scheduler Web UI - -Run the following command to redirect localhost port 8080 to port 80 in the scheduler container and then view the scheduler UI at http://localhost:8080. - -```shell -kubectl port-forward ballista-scheduler-0 8080:80 -``` - -## Connect - -```shell -kubectl port-forward ballista-scheduler-0 50050:50050 -sqline # ... see FlightSQL instructions -``` - -## Teardown - -```shell -helm uninstall ballista -``` \ No newline at end of file diff --git a/helm/ballista/.gitignore b/helm/ballista/.gitignore deleted file mode 100644 index f263fee85..000000000 --- a/helm/ballista/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.tar.gz -*.tgz - diff --git a/helm/ballista/.helmignore b/helm/ballista/.helmignore deleted file mode 100644 index bd2d4649d..000000000 --- a/helm/ballista/.helmignore +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/ballista/Chart.lock b/helm/ballista/Chart.lock deleted file mode 100644 index 62875df04..000000000 --- a/helm/ballista/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: minio - repository: https://charts.bitnami.com/bitnami - version: 11.10.5 -digest: sha256:53448ec4e280c4a77c33fe944da96c38bb877f3777c9a9dcfc364fc9012a1b09 -generated: "2022-10-07T13:05:49.084667348-06:00" diff --git a/helm/ballista/Chart.yaml b/helm/ballista/Chart.yaml deleted file mode 100644 index 913a954a1..000000000 --- a/helm/ballista/Chart.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -apiVersion: v2 -name: ballista -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -dependencies: - - name: minio - version: 11.10.x - repository: https://charts.bitnami.com/bitnami diff --git a/helm/ballista/templates/NOTES.txt b/helm/ballista/templates/NOTES.txt deleted file mode 100644 index a63344279..000000000 --- a/helm/ballista/templates/NOTES.txt +++ /dev/null @@ -1,40 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.scheduler.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "ballista.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.scheduler.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "ballista.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "ballista.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.scheduler.port }} -{{- else if contains "ClusterIP" .Values.service.scheduler.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "ballista.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit 127.0.0.1:50050 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 50050:$CONTAINER_PORT -{{- end }} diff --git a/helm/ballista/templates/_helpers.tpl b/helm/ballista/templates/_helpers.tpl deleted file mode 100644 index e575aaa6d..000000000 --- a/helm/ballista/templates/_helpers.tpl +++ /dev/null @@ -1,77 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} -{{- define "ballista.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "ballista.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "ballista.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "ballista.labels" -}} -helm.sh/chart: {{ include "ballista.chart" . }} -{{ include "ballista.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "ballista.selectorLabels" -}} -app.kubernetes.io/name: {{ include "ballista.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "ballista.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "ballista.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm/ballista/templates/executor.yaml b/helm/ballista/templates/executor.yaml deleted file mode 100644 index 4408d175c..000000000 --- a/helm/ballista/templates/executor.yaml +++ /dev/null @@ -1,107 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: ballista-executor - labels: - {{- include "ballista.labels" . | nindent 4 }} -spec: - serviceName: executor - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - run: ballista-executor - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - run: ballista-executor - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "ballista.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }}-executor - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}/{{ .Values.image.executor }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/root/ballista-executor", "--scheduler-host=ballista-scheduler"] - env: - - name: AWS_DEFAULT_REGION - value: {{.Values.config.s3.region}} - - name: AWS_ENDPOINT - value: {{.Values.config.s3.endpoint}} - - name: AWS_ALLOW_HTTP - value: {{.Values.config.s3.allowHttp | quote}} - - name: AWS_ACCESS_KEY_ID - value: {{.Values.minio.auth.rootUser}} - - name: AWS_SECRET_ACCESS_KEY - value: {{.Values.minio.auth.rootPassword}} - ports: - - name: flight - containerPort: 50051 - protocol: TCP - livenessProbe: - tcpSocket: - port: 50051 - readinessProbe: - tcpSocket: - port: 50051 - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - ---- -apiVersion: v1 -kind: Service -metadata: - name: ballista-executor - labels: - run: ballista-executor -spec: - type: {{ .Values.service.executor.type }} - ports: - - port: {{ .Values.service.executor.port }} - targetPort: 50051 - protocol: TCP - name: flight - selector: - run: ballista-executor diff --git a/helm/ballista/templates/hpa.yaml b/helm/ballista/templates/hpa.yaml deleted file mode 100644 index d83ab7d26..000000000 --- a/helm/ballista/templates/hpa.yaml +++ /dev/null @@ -1,46 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "ballista.fullname" . }} - labels: - {{- include "ballista.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "ballista.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/ballista/templates/ingress.yaml b/helm/ballista/templates/ingress.yaml deleted file mode 100644 index e1f56c109..000000000 --- a/helm/ballista/templates/ingress.yaml +++ /dev/null @@ -1,79 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "ballista.fullname" . -}} -{{- $svcPort := .Values.service.scheduler.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "ballista.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/ballista/templates/scheduler.yaml b/helm/ballista/templates/scheduler.yaml deleted file mode 100644 index fc44c2f20..000000000 --- a/helm/ballista/templates/scheduler.yaml +++ /dev/null @@ -1,106 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: ballista-scheduler - labels: - {{- include "ballista.labels" . | nindent 4 }} -spec: - serviceName: scheduler - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - run: ballista-scheduler - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - run: ballista-scheduler - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "ballista.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }}-scheduler - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}/{{ .Values.image.scheduler }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: AWS_DEFAULT_REGION - value: {{.Values.config.s3.region}} - - name: AWS_ENDPOINT - value: {{.Values.config.s3.endpoint}} - - name: AWS_ALLOW_HTTP - value: {{.Values.config.s3.allowHttp | quote}} - - name: AWS_ACCESS_KEY_ID - value: {{.Values.minio.auth.rootUser}} - - name: AWS_SECRET_ACCESS_KEY - value: {{.Values.minio.auth.rootPassword}} - ports: - - name: flightsql - containerPort: 50050 - protocol: TCP - livenessProbe: - tcpSocket: - port: 50050 - readinessProbe: - tcpSocket: - port: 50050 - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - ---- -apiVersion: v1 -kind: Service -metadata: - name: ballista-scheduler - labels: - run: ballista-scheduler -spec: - type: {{ .Values.service.scheduler.type }} - ports: - - port: {{ .Values.service.scheduler.port }} - targetPort: 50050 - protocol: TCP - name: flightsql - selector: - run: ballista-scheduler diff --git a/helm/ballista/templates/serviceaccount.yaml b/helm/ballista/templates/serviceaccount.yaml deleted file mode 100644 index 6f3ae78be..000000000 --- a/helm/ballista/templates/serviceaccount.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "ballista.serviceAccountName" . }} - labels: - {{- include "ballista.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/helm/ballista/templates/tests/test-connection.yaml b/helm/ballista/templates/tests/test-connection.yaml deleted file mode 100644 index 136492b33..000000000 --- a/helm/ballista/templates/tests/test-connection.yaml +++ /dev/null @@ -1,33 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "ballista.fullname" . }}-test-connection" - labels: - {{- include "ballista.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "ballista.fullname" . }}:{{ .Values.service.scheduler.port }}'] - restartPolicy: Never diff --git a/helm/ballista/values.yaml b/helm/ballista/values.yaml deleted file mode 100644 index 1bae89341..000000000 --- a/helm/ballista/values.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -replicaCount: 1 - -image: - repository: "" - scheduler: ballista-scheduler - executor: ballista-executor - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "latest" - -config: - s3: - allowHttp: true - region: "minio" - endpoint: "http://ballista-minio:9000" - -imagePullSecrets: - - name: regcred -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - scheduler: - type: ClusterIP - port: 50050 - executor: - type: ClusterIP - port: 50051 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: ballista-scheduler.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -minio: - persistence: - enabled: false - storageClass: azurefile-csi - size: 1Gi - existingClaim: "" - auth: - rootUser: minio From bb5388b425146fa57932a352ca74e96ac2c3d5f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Sat, 12 Oct 2024 00:28:13 +0100 Subject: [PATCH 2/5] remove plugin subsystem ... (#1070) ... as it was never finished --- ballista/core/Cargo.toml | 1 - ballista/core/src/lib.rs | 2 - ballista/core/src/plugin/mod.rs | 128 ----------------- ballista/core/src/plugin/plugin_manager.rs | 150 -------------------- ballista/core/src/plugin/udf.rs | 151 --------------------- 5 files changed, 432 deletions(-) delete mode 100644 ballista/core/src/plugin/mod.rs delete mode 100644 ballista/core/src/plugin/plugin_manager.rs delete mode 100644 ballista/core/src/plugin/udf.rs diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml index 8a01f56fb..daabe8a27 100644 --- a/ballista/core/Cargo.toml +++ b/ballista/core/Cargo.toml @@ -60,7 +60,6 @@ futures = "0.3" hashbrown = "0.14" itertools = "0.12" -libloading = "0.8.0" log = "0.4" md-5 = { version = "^0.10.0" } object_store = { workspace = true } diff --git a/ballista/core/src/lib.rs b/ballista/core/src/lib.rs index 301bc0507..5306e8b98 100644 --- a/ballista/core/src/lib.rs +++ b/ballista/core/src/lib.rs @@ -31,8 +31,6 @@ pub mod error; pub mod event_loop; pub mod execution_plans; pub mod object_store_registry; -/// some plugins -pub mod plugin; pub mod utils; #[macro_use] diff --git a/ballista/core/src/plugin/mod.rs b/ballista/core/src/plugin/mod.rs deleted file mode 100644 index 8d2cd6b9b..000000000 --- a/ballista/core/src/plugin/mod.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::error::Result; -use crate::plugin::udf::UDFPluginManager; -use libloading::Library; -use std::any::Any; -use std::env; -use std::sync::Arc; - -/// plugin manager -pub mod plugin_manager; -/// udf plugin -pub mod udf; - -/// CARGO_PKG_VERSION -pub static CORE_VERSION: &str = env!("CARGO_PKG_VERSION"); -/// RUSTC_VERSION -pub static RUSTC_VERSION: &str = env!("RUSTC_VERSION"); - -/// Top plugin trait -pub trait Plugin { - /// Returns the plugin as [`Any`](std::any::Any) so that it can be - /// downcast to a specific implementation. - fn as_any(&self) -> &dyn Any; -} - -/// The enum of Plugin -#[repr(C)] -#[derive(PartialEq, std::cmp::Eq, std::hash::Hash, Copy, Clone)] -pub enum PluginEnum { - /// UDF/UDAF plugin - UDF, -} - -impl PluginEnum { - /// new a struct which impl the PluginRegistrar trait - pub fn init_plugin_manager(&self) -> Box { - match self { - PluginEnum::UDF => Box::::default(), - } - } -} - -/// Every plugin need a PluginDeclaration -#[derive(Copy, Clone)] -pub struct PluginDeclaration { - /// Rust doesn’t have a stable ABI, meaning different compiler versions can generate incompatible code. - /// For these reasons, the UDF plug-in must be compiled using the same version of rustc as datafusion. - pub rustc_version: &'static str, - - /// core version of the plugin. The plugin's core_version need same as plugin manager. - pub core_version: &'static str, - - /// One of PluginEnum - pub plugin_type: unsafe extern "C" fn() -> PluginEnum, -} - -/// Plugin Registrar , Every plugin need implement this trait -pub trait PluginRegistrar: Send + Sync + 'static { - /// # Safety - /// load plugin from library - unsafe fn load(&mut self, library: Arc) -> Result<()>; - - /// Returns the plugin as [`Any`](std::any::Any) so that it can be - /// downcast to a specific implementation. - fn as_any(&self) -> &dyn Any; -} - -/// Declare a plugin's PluginDeclaration. -/// -/// # Notes -/// -/// This works by automatically generating an `extern "C"` function named `get_plugin_type` with a -/// pre-defined signature and symbol name. And then generating a PluginDeclaration. -/// Therefore you will only be able to declare one plugin per library. -#[macro_export] -macro_rules! declare_plugin { - ($plugin_type:expr) => { - #[no_mangle] - pub extern "C" fn get_plugin_type() -> $crate::plugin::PluginEnum { - $plugin_type - } - - #[no_mangle] - pub static plugin_declaration: $crate::plugin::PluginDeclaration = - $crate::plugin::PluginDeclaration { - rustc_version: $crate::plugin::RUSTC_VERSION, - core_version: $crate::plugin::CORE_VERSION, - plugin_type: get_plugin_type, - }; - }; -} - -/// get the plugin dir -pub fn plugin_dir() -> String { - let current_exe_dir = match env::current_exe() { - Ok(exe_path) => exe_path.display().to_string(), - Err(_e) => "".to_string(), - }; - - // If current_exe_dir contain `deps` the root dir is the parent dir - // eg: /Users/xxx/workspace/rust_plugin_sty/target/debug/deps/plugins_app-067452b3ff2af70e - // the plugin dir is /Users/xxx/workspace/rust_plugin_sty/target/debug/deps - // else eg: /Users/xxx/workspace/rust_plugin_sty/target/debug/plugins_app - // the plugin dir is /Users/xxx/workspace/rust_plugin_sty/target/debug/ - if current_exe_dir.contains("/deps/") { - let i = current_exe_dir.find("/deps/").unwrap(); - String::from(¤t_exe_dir.as_str()[..i + 6]) - } else { - let i = current_exe_dir.rfind('/').unwrap(); - String::from(¤t_exe_dir.as_str()[..i]) - } -} diff --git a/ballista/core/src/plugin/plugin_manager.rs b/ballista/core/src/plugin/plugin_manager.rs deleted file mode 100644 index 6c19f0542..000000000 --- a/ballista/core/src/plugin/plugin_manager.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -use crate::error::{BallistaError, Result}; -use libloading::Library; -use log::info; -use std::collections::HashMap; -use std::io; -use std::sync::{Arc, Mutex}; -use walkdir::{DirEntry, WalkDir}; - -use crate::plugin::{ - PluginDeclaration, PluginEnum, PluginRegistrar, CORE_VERSION, RUSTC_VERSION, -}; -use once_cell::sync::OnceCell; - -/// To prevent the library from being loaded multiple times, we use once_cell defines a Arc> -/// Because datafusion is a library, not a service, users may not need to load all plug-ins in the process. -/// So fn global_plugin_manager return Arc>. In this way, users can load the required library through the load method of GlobalPluginManager when needed -static INSTANCE: OnceCell>> = OnceCell::new(); - -/// global_plugin_manager -pub fn global_plugin_manager( - plugin_path: &str, -) -> &'static Arc> { - INSTANCE.get_or_init(move || unsafe { - let mut gpm = GlobalPluginManager::default(); - gpm.load(plugin_path).unwrap(); - Arc::new(Mutex::new(gpm)) - }) -} - -#[derive(Default)] -/// manager all plugin_type's plugin_manager -pub struct GlobalPluginManager { - /// every plugin need a plugin registrar - pub plugin_managers: HashMap>, - - /// loaded plugin files - pub plugin_files: Vec, -} - -impl GlobalPluginManager { - /// # Safety - /// find plugin file from `plugin_path` and load it . - unsafe fn load(&mut self, plugin_path: &str) -> Result<()> { - if "".eq(plugin_path) { - return Ok(()); - } - // find library file from udaf_plugin_path - info!("load plugin from dir:{}", plugin_path); - - let plugin_files = self.get_all_plugin_files(plugin_path)?; - - for plugin_file in plugin_files { - let library = Library::new(plugin_file.path()).map_err(|e| { - BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - format!("load library error: {e}"), - )) - })?; - - let library = Arc::new(library); - - let dec = library.get::<*mut PluginDeclaration>(b"plugin_declaration\0"); - if dec.is_err() { - info!( - "not found plugin_declaration in the library: {}", - plugin_file.path().to_str().unwrap() - ); - continue; - } - - let dec = dec.unwrap().read(); - - // ersion checks to prevent accidental ABI incompatibilities - if dec.rustc_version != RUSTC_VERSION || dec.core_version != CORE_VERSION { - return Err(BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - "Version mismatch", - ))); - } - - let plugin_enum = (dec.plugin_type)(); - let curr_plugin_manager = match self.plugin_managers.get_mut(&plugin_enum) { - None => { - let plugin_manager = plugin_enum.init_plugin_manager(); - self.plugin_managers.insert(plugin_enum, plugin_manager); - self.plugin_managers.get_mut(&plugin_enum).unwrap() - } - Some(manager) => manager, - }; - curr_plugin_manager.load(library)?; - self.plugin_files - .push(plugin_file.path().to_str().unwrap().to_string()); - } - - Ok(()) - } - - /// get all plugin file in the dir - fn get_all_plugin_files(&self, plugin_path: &str) -> io::Result> { - let mut plugin_files = Vec::new(); - for entry in WalkDir::new(plugin_path).into_iter().filter_map(|e| { - let item = e.unwrap(); - // every file only load once - if self - .plugin_files - .contains(&item.path().to_str().unwrap().to_string()) - { - return None; - } - - let file_type = item.file_type(); - if !file_type.is_file() { - return None; - } - - if let Some(path) = item.path().extension() { - if let Some(suffix) = path.to_str() { - if suffix == "dylib" || suffix == "so" || suffix == "dll" { - info!( - "load plugin from library file:{}", - item.path().to_str().unwrap() - ); - return Some(item); - } - } - } - - None - }) { - plugin_files.push(entry); - } - Ok(plugin_files) - } -} diff --git a/ballista/core/src/plugin/udf.rs b/ballista/core/src/plugin/udf.rs deleted file mode 100644 index 36ebd5e32..000000000 --- a/ballista/core/src/plugin/udf.rs +++ /dev/null @@ -1,151 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -use crate::error::{BallistaError, Result}; -use crate::plugin::plugin_manager::global_plugin_manager; -use crate::plugin::{Plugin, PluginEnum, PluginRegistrar}; -use datafusion::logical_expr::{AggregateUDF, ScalarUDF}; -use libloading::{Library, Symbol}; -use std::any::Any; -use std::collections::HashMap; -use std::io; -use std::sync::Arc; - -/// UDF plugin trait -pub trait UDFPlugin: Plugin { - /// get a ScalarUDF by name - fn get_scalar_udf_by_name(&self, fun_name: &str) -> Result; - - /// return all udf names in the plugin - fn udf_names(&self) -> Result>; - - /// get a aggregate udf by name - fn get_aggregate_udf_by_name(&self, fun_name: &str) -> Result; - - /// return all udaf names - fn udaf_names(&self) -> Result>; -} - -/// UDFPluginManager -#[derive(Default, Clone)] -pub struct UDFPluginManager { - /// scalar udfs - pub scalar_udfs: HashMap>, - - /// aggregate udfs - pub aggregate_udfs: HashMap>, - - /// All libraries load from the plugin dir. - pub libraries: Vec>, -} - -impl PluginRegistrar for UDFPluginManager { - unsafe fn load(&mut self, library: Arc) -> Result<()> { - type PluginRegister = unsafe fn() -> Box; - let register_fun: Symbol = - library.get(b"registrar_udf_plugin\0").map_err(|e| { - BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - format!("not found fn registrar_udf_plugin in the library: {e}"), - )) - })?; - - let udf_plugin: Box = register_fun(); - udf_plugin - .udf_names() - .unwrap() - .iter() - .try_for_each(|udf_name| { - if self.scalar_udfs.contains_key(udf_name) { - Err(BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - format!("udf name: {udf_name} already exists"), - ))) - } else { - let scalar_udf = udf_plugin.get_scalar_udf_by_name(udf_name)?; - self.scalar_udfs - .insert(udf_name.to_string(), Arc::new(scalar_udf)); - Ok(()) - } - })?; - - udf_plugin - .udaf_names() - .unwrap() - .iter() - .try_for_each(|udaf_name| { - if self.aggregate_udfs.contains_key(udaf_name) { - Err(BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - format!("udaf name: {udaf_name} already exists"), - ))) - } else { - let aggregate_udf = - udf_plugin.get_aggregate_udf_by_name(udaf_name)?; - self.aggregate_udfs - .insert(udaf_name.to_string(), Arc::new(aggregate_udf)); - Ok(()) - } - })?; - self.libraries.push(library); - Ok(()) - } - - fn as_any(&self) -> &dyn Any { - self - } -} - -/// Declare a udf plugin registrar callback -/// -/// # Notes -/// -/// This works by automatically generating an `extern "C"` function named `registrar_udf_plugin` with a -/// pre-defined signature and symbol name. -/// Therefore you will only be able to declare one plugin per library. -#[macro_export] -macro_rules! declare_udf_plugin { - ($curr_plugin_type:ty, $constructor:path) => { - #[no_mangle] - pub extern "C" fn registrar_udf_plugin() -> Box { - // make sure the constructor is the correct type. - let constructor: fn() -> $curr_plugin_type = $constructor; - let object = constructor(); - Box::new(object) - } - - $crate::declare_plugin!($crate::plugin::PluginEnum::UDF); - }; -} - -/// get a Option of Immutable UDFPluginManager -pub fn get_udf_plugin_manager(path: &str) -> Option { - let udf_plugin_manager_opt = { - let gpm = global_plugin_manager(path).lock().unwrap(); - let plugin_registrar_opt = gpm.plugin_managers.get(&PluginEnum::UDF); - if let Some(plugin_registrar) = plugin_registrar_opt { - if let Some(udf_plugin_manager) = - plugin_registrar.as_any().downcast_ref::() - { - return Some(udf_plugin_manager.clone()); - } else { - return None; - } - } - None - }; - udf_plugin_manager_opt -} From 1591fed34429f30942522d1c00118f1370c99c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Sat, 12 Oct 2024 00:28:25 +0100 Subject: [PATCH 3/5] Remove CI folder (#1074) it looks like not used anywhere. Relates to: #1066 & #1067 --- ci/appveyor-cpp-build.bat | 162 -------- ci/appveyor-cpp-setup.bat | 101 ----- ci/conda_env_archery.yml | 42 -- ci/conda_env_cpp.yml | 42 -- ci/conda_env_crossbow.txt | 25 -- ci/conda_env_gandiva.yml | 19 - ci/conda_env_gandiva_win.yml | 20 - ci/conda_env_python.yml | 32 -- ci/conda_env_r.yml | 37 -- ci/conda_env_sphinx.yml | 24 -- ci/conda_env_unix.yml | 23 -- ci/detect-changes.py | 365 ------------------ ci/docker/conda-cpp.Dockerfile | 54 --- ci/docker/conda-integration.Dockerfile | 69 ---- ci/docker/conda-python-dask.Dockerfile | 25 -- ci/docker/conda-python-hdfs.Dockerfile | 52 --- ci/docker/conda-python-jpype.Dockerfile | 29 -- ci/docker/conda-python-kartothek.Dockerfile | 46 --- ci/docker/conda-python-pandas.Dockerfile | 27 -- ci/docker/conda-python-spark.Dockerfile | 43 --- ci/docker/conda-python-turbodbc.Dockerfile | 50 --- ci/docker/conda-python.Dockerfile | 37 -- ci/docker/conda.Dockerfile | 53 --- ci/docker/debian-10-cpp.Dockerfile | 104 ----- ci/docker/debian-10-go.Dockerfile | 25 -- ci/docker/debian-10-js.Dockerfile | 27 -- ci/docker/debian-9-java.Dockerfile | 28 -- ci/docker/fedora-33-cpp.Dockerfile | 92 ----- ci/docker/linux-apt-c-glib.Dockerfile | 65 ---- ci/docker/linux-apt-docs.Dockerfile | 107 ----- ci/docker/linux-apt-jni.Dockerfile | 86 ----- ci/docker/linux-apt-lint.Dockerfile | 69 ---- ci/docker/linux-apt-python-3.Dockerfile | 46 --- ci/docker/linux-apt-r.Dockerfile | 100 ----- ci/docker/linux-apt-ruby.Dockerfile | 27 -- ci/docker/linux-dnf-python-3.Dockerfile | 41 -- ci/docker/linux-r.Dockerfile | 44 --- ci/docker/python-sdist.Dockerfile | 36 -- .../python-wheel-manylinux-201x.Dockerfile | 110 ------ .../python-wheel-manylinux-test.Dockerfile | 27 -- .../python-wheel-windows-vs2017.Dockerfile | 99 ----- ci/docker/ubuntu-18.04-cpp.Dockerfile | 128 ------ ci/docker/ubuntu-18.04-csharp.Dockerfile | 22 -- ci/docker/ubuntu-20.04-cpp.Dockerfile | 135 ------- ci/docker/ubuntu-20.10-cpp.Dockerfile | 137 ------- ci/etc/hdfs-site.xml | 52 --- ci/etc/rprofile | 53 --- ci/scripts/PKGBUILD | 134 ------- ci/scripts/c_glib_build.sh | 45 --- ci/scripts/c_glib_test.sh | 48 --- ci/scripts/ccache_setup.sh | 26 -- ci/scripts/cpp_build.sh | 156 -------- ci/scripts/cpp_test.sh | 95 ----- ci/scripts/csharp_build.sh | 26 -- ci/scripts/csharp_pack.sh | 26 -- ci/scripts/csharp_test.sh | 29 -- ci/scripts/docs_build.sh | 48 --- ci/scripts/go_build.sh | 36 -- ci/scripts/go_test.sh | 38 -- ci/scripts/install_conda.sh | 66 ---- ci/scripts/install_dask.sh | 36 -- ci/scripts/install_iwyu.sh | 48 --- ci/scripts/install_kartothek.sh | 41 -- ci/scripts/install_minio.sh | 52 --- ci/scripts/install_osx_sdk.sh | 41 -- ci/scripts/install_pandas.sh | 46 --- ci/scripts/install_spark.sh | 31 -- ci/scripts/install_turbodbc.sh | 37 -- ci/scripts/integration_arrow.sh | 36 -- ci/scripts/integration_dask.sh | 42 -- ci/scripts/integration_hdfs.sh | 69 ---- ci/scripts/integration_hiveserver2.sh | 32 -- ci/scripts/integration_kartothek.sh | 31 -- ci/scripts/integration_spark.sh | 87 ----- ci/scripts/integration_turbodbc.sh | 47 --- ci/scripts/java_build.sh | 101 ----- ci/scripts/java_test.sh | 49 --- ci/scripts/js_build.sh | 36 -- ci/scripts/js_test.sh | 29 -- ci/scripts/msys2_setup.sh | 69 ---- ci/scripts/msys2_system_clean.sh | 33 -- ci/scripts/msys2_system_upgrade.sh | 48 --- ci/scripts/python_benchmark.sh | 40 -- ci/scripts/python_build.sh | 54 --- ci/scripts/python_sdist_build.sh | 27 -- ci/scripts/python_sdist_test.sh | 52 --- ci/scripts/python_test.sh | 32 -- ci/scripts/python_wheel_macos_build.sh | 133 ------- ci/scripts/python_wheel_macos_test.sh | 66 ---- ci/scripts/python_wheel_manylinux_build.sh | 141 ------- ci/scripts/python_wheel_manylinux_test.sh | 73 ---- ci/scripts/python_wheel_windows_build.bat | 109 ------ ci/scripts/python_wheel_windows_test.bat | 54 --- ci/scripts/r_build.sh | 33 -- ci/scripts/r_deps.sh | 31 -- ci/scripts/r_docker_configure.sh | 80 ---- ci/scripts/r_pkgdown_check.sh | 41 -- ci/scripts/r_sanitize.sh | 36 -- ci/scripts/r_test.sh | 88 ----- ci/scripts/r_windows_build.sh | 102 ----- ci/scripts/release_test.sh | 28 -- ci/scripts/ruby_test.sh | 29 -- ci/scripts/util_checkout.sh | 28 -- ci/scripts/util_cleanup.sh | 54 --- ci/scripts/util_download_apache.sh | 53 --- ci/scripts/util_wait_for_it.sh | 199 ---------- ci/vcpkg/arm64-linux-static-debug.cmake | 28 -- ci/vcpkg/arm64-linux-static-release.cmake | 28 -- ci/vcpkg/ports.patch | 63 --- ci/vcpkg/x64-linux-static-debug.cmake | 24 -- ci/vcpkg/x64-linux-static-release.cmake | 24 -- ci/vcpkg/x64-osx-static-debug.cmake | 25 -- ci/vcpkg/x64-osx-static-release.cmake | 25 -- ci/vcpkg/x64-windows-static-md-debug.cmake | 22 -- ci/vcpkg/x64-windows-static-md-release.cmake | 22 -- 115 files changed, 6695 deletions(-) delete mode 100644 ci/appveyor-cpp-build.bat delete mode 100644 ci/appveyor-cpp-setup.bat delete mode 100644 ci/conda_env_archery.yml delete mode 100644 ci/conda_env_cpp.yml delete mode 100644 ci/conda_env_crossbow.txt delete mode 100644 ci/conda_env_gandiva.yml delete mode 100644 ci/conda_env_gandiva_win.yml delete mode 100644 ci/conda_env_python.yml delete mode 100644 ci/conda_env_r.yml delete mode 100644 ci/conda_env_sphinx.yml delete mode 100644 ci/conda_env_unix.yml delete mode 100644 ci/detect-changes.py delete mode 100644 ci/docker/conda-cpp.Dockerfile delete mode 100644 ci/docker/conda-integration.Dockerfile delete mode 100644 ci/docker/conda-python-dask.Dockerfile delete mode 100644 ci/docker/conda-python-hdfs.Dockerfile delete mode 100644 ci/docker/conda-python-jpype.Dockerfile delete mode 100644 ci/docker/conda-python-kartothek.Dockerfile delete mode 100644 ci/docker/conda-python-pandas.Dockerfile delete mode 100644 ci/docker/conda-python-spark.Dockerfile delete mode 100644 ci/docker/conda-python-turbodbc.Dockerfile delete mode 100644 ci/docker/conda-python.Dockerfile delete mode 100644 ci/docker/conda.Dockerfile delete mode 100644 ci/docker/debian-10-cpp.Dockerfile delete mode 100644 ci/docker/debian-10-go.Dockerfile delete mode 100644 ci/docker/debian-10-js.Dockerfile delete mode 100644 ci/docker/debian-9-java.Dockerfile delete mode 100644 ci/docker/fedora-33-cpp.Dockerfile delete mode 100644 ci/docker/linux-apt-c-glib.Dockerfile delete mode 100644 ci/docker/linux-apt-docs.Dockerfile delete mode 100644 ci/docker/linux-apt-jni.Dockerfile delete mode 100644 ci/docker/linux-apt-lint.Dockerfile delete mode 100644 ci/docker/linux-apt-python-3.Dockerfile delete mode 100644 ci/docker/linux-apt-r.Dockerfile delete mode 100644 ci/docker/linux-apt-ruby.Dockerfile delete mode 100644 ci/docker/linux-dnf-python-3.Dockerfile delete mode 100644 ci/docker/linux-r.Dockerfile delete mode 100644 ci/docker/python-sdist.Dockerfile delete mode 100644 ci/docker/python-wheel-manylinux-201x.Dockerfile delete mode 100644 ci/docker/python-wheel-manylinux-test.Dockerfile delete mode 100644 ci/docker/python-wheel-windows-vs2017.Dockerfile delete mode 100644 ci/docker/ubuntu-18.04-cpp.Dockerfile delete mode 100644 ci/docker/ubuntu-18.04-csharp.Dockerfile delete mode 100644 ci/docker/ubuntu-20.04-cpp.Dockerfile delete mode 100644 ci/docker/ubuntu-20.10-cpp.Dockerfile delete mode 100644 ci/etc/hdfs-site.xml delete mode 100644 ci/etc/rprofile delete mode 100644 ci/scripts/PKGBUILD delete mode 100755 ci/scripts/c_glib_build.sh delete mode 100755 ci/scripts/c_glib_test.sh delete mode 100755 ci/scripts/ccache_setup.sh delete mode 100755 ci/scripts/cpp_build.sh delete mode 100755 ci/scripts/cpp_test.sh delete mode 100755 ci/scripts/csharp_build.sh delete mode 100755 ci/scripts/csharp_pack.sh delete mode 100755 ci/scripts/csharp_test.sh delete mode 100755 ci/scripts/docs_build.sh delete mode 100755 ci/scripts/go_build.sh delete mode 100755 ci/scripts/go_test.sh delete mode 100755 ci/scripts/install_conda.sh delete mode 100755 ci/scripts/install_dask.sh delete mode 100755 ci/scripts/install_iwyu.sh delete mode 100755 ci/scripts/install_kartothek.sh delete mode 100755 ci/scripts/install_minio.sh delete mode 100755 ci/scripts/install_osx_sdk.sh delete mode 100755 ci/scripts/install_pandas.sh delete mode 100755 ci/scripts/install_spark.sh delete mode 100755 ci/scripts/install_turbodbc.sh delete mode 100755 ci/scripts/integration_arrow.sh delete mode 100755 ci/scripts/integration_dask.sh delete mode 100755 ci/scripts/integration_hdfs.sh delete mode 100755 ci/scripts/integration_hiveserver2.sh delete mode 100755 ci/scripts/integration_kartothek.sh delete mode 100755 ci/scripts/integration_spark.sh delete mode 100755 ci/scripts/integration_turbodbc.sh delete mode 100755 ci/scripts/java_build.sh delete mode 100755 ci/scripts/java_test.sh delete mode 100755 ci/scripts/js_build.sh delete mode 100755 ci/scripts/js_test.sh delete mode 100755 ci/scripts/msys2_setup.sh delete mode 100755 ci/scripts/msys2_system_clean.sh delete mode 100755 ci/scripts/msys2_system_upgrade.sh delete mode 100755 ci/scripts/python_benchmark.sh delete mode 100755 ci/scripts/python_build.sh delete mode 100755 ci/scripts/python_sdist_build.sh delete mode 100755 ci/scripts/python_sdist_test.sh delete mode 100755 ci/scripts/python_test.sh delete mode 100755 ci/scripts/python_wheel_macos_build.sh delete mode 100755 ci/scripts/python_wheel_macos_test.sh delete mode 100755 ci/scripts/python_wheel_manylinux_build.sh delete mode 100755 ci/scripts/python_wheel_manylinux_test.sh delete mode 100644 ci/scripts/python_wheel_windows_build.bat delete mode 100755 ci/scripts/python_wheel_windows_test.bat delete mode 100755 ci/scripts/r_build.sh delete mode 100755 ci/scripts/r_deps.sh delete mode 100755 ci/scripts/r_docker_configure.sh delete mode 100755 ci/scripts/r_pkgdown_check.sh delete mode 100755 ci/scripts/r_sanitize.sh delete mode 100755 ci/scripts/r_test.sh delete mode 100755 ci/scripts/r_windows_build.sh delete mode 100755 ci/scripts/release_test.sh delete mode 100755 ci/scripts/ruby_test.sh delete mode 100755 ci/scripts/util_checkout.sh delete mode 100755 ci/scripts/util_cleanup.sh delete mode 100755 ci/scripts/util_download_apache.sh delete mode 100755 ci/scripts/util_wait_for_it.sh delete mode 100644 ci/vcpkg/arm64-linux-static-debug.cmake delete mode 100644 ci/vcpkg/arm64-linux-static-release.cmake delete mode 100644 ci/vcpkg/ports.patch delete mode 100644 ci/vcpkg/x64-linux-static-debug.cmake delete mode 100644 ci/vcpkg/x64-linux-static-release.cmake delete mode 100644 ci/vcpkg/x64-osx-static-debug.cmake delete mode 100644 ci/vcpkg/x64-osx-static-release.cmake delete mode 100644 ci/vcpkg/x64-windows-static-md-debug.cmake delete mode 100644 ci/vcpkg/x64-windows-static-md-release.cmake diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat deleted file mode 100644 index 6b9309396..000000000 --- a/ci/appveyor-cpp-build.bat +++ /dev/null @@ -1,162 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@echo on - -git config core.symlinks true -git reset --hard - -@rem Retrieve git submodules, configure env var for Parquet unit tests -git submodule update --init || exit /B - -set ARROW_TEST_DATA=%CD%\testing\data -set PARQUET_TEST_DATA=%CD%\cpp\submodules\parquet-testing\data - -@rem -@rem In the configurations below we disable building the Arrow static library -@rem to save some time. Unfortunately this will still build the Parquet static -@rem library because of PARQUET-1420 (Thrift-generated symbols not exported in DLL). -@rem -if "%JOB%" == "Build_Debug" ( - mkdir cpp\build-debug - pushd cpp\build-debug - - cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=OFF ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DARROW_BUILD_STATIC=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_CXXFLAGS="/MP" ^ - -DARROW_ENABLE_TIMING_TESTS=OFF ^ - -DARROW_USE_PRECOMPILED_HEADERS=OFF ^ - -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^ - -DCMAKE_BUILD_TYPE="Debug" ^ - -DCMAKE_UNITY_BUILD=ON ^ - .. || exit /B - - cmake --build . --config Debug || exit /B - ctest --output-on-failure -j2 || exit /B - popd - - @rem Finish Debug build successfully - exit /B 0 -) - -call activate arrow - -@rem Use Boost from Anaconda -set BOOST_ROOT=%CONDA_PREFIX%\Library -set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib - -@rem The "main" C++ build script for Windows CI -@rem (i.e. for usual configurations) - -if "%JOB%" == "Toolchain" ( - set CMAKE_ARGS=-DARROW_DEPENDENCY_SOURCE=CONDA -DARROW_WITH_BZ2=ON -) else ( - @rem We're in a conda environment but don't want to use it for the dependencies - set CMAKE_ARGS=-DARROW_DEPENDENCY_SOURCE=AUTO -) - -@rem Enable warnings-as-errors -set ARROW_CXXFLAGS=/WX /MP - -@rem -@rem Build and test Arrow C++ libraries (including Parquet) -@rem - -mkdir cpp\build -pushd cpp\build - -@rem XXX Without forcing CMAKE_CXX_COMPILER, CMake can re-run itself and -@rem unfortunately switch from Release to Debug mode... -@rem -@rem In release mode, disable optimizations (/Od) for faster compiling -@rem and enable runtime assertions. - -cmake -G "%GENERATOR%" %CMAKE_ARGS% ^ - -DARROW_BOOST_USE_SHARED=ON ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DARROW_BUILD_STATIC=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^ - -DARROW_DATASET=ON ^ - -DARROW_ENABLE_TIMING_TESTS=OFF ^ - -DARROW_FLIGHT=%ARROW_BUILD_FLIGHT% ^ - -DARROW_GANDIVA=%ARROW_BUILD_GANDIVA% ^ - -DARROW_MIMALLOC=ON ^ - -DARROW_PARQUET=ON ^ - -DARROW_PYTHON=ON ^ - -DARROW_S3=%ARROW_S3% ^ - -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^ - -DARROW_WITH_BROTLI=ON ^ - -DARROW_WITH_LZ4=ON ^ - -DARROW_WITH_SNAPPY=ON ^ - -DARROW_WITH_ZLIB=ON ^ - -DARROW_WITH_ZSTD=ON ^ - -DCMAKE_BUILD_TYPE="Release" ^ - -DCMAKE_CXX_COMPILER=clcache ^ - -DCMAKE_CXX_FLAGS_RELEASE="/MD /Od /UNDEBUG" ^ - -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ - -DCMAKE_UNITY_BUILD=ON ^ - -DCMAKE_VERBOSE_MAKEFILE=OFF ^ - -DPARQUET_BUILD_EXECUTABLES=ON ^ - -DPARQUET_REQUIRE_ENCRYPTION=ON ^ - .. || exit /B -cmake --build . --target install --config %CONFIGURATION% || exit /B - -@rem Needed so arrow-python-test.exe works -set OLD_PYTHONHOME=%PYTHONHOME% -set PYTHONHOME=%CONDA_PREFIX% - -ctest --output-on-failure -j2 || exit /B - -set PYTHONHOME=%OLD_PYTHONHOME% -popd - -@rem -@rem Build and install pyarrow -@rem - -pushd python - -set PYARROW_BUNDLE_BOOST=OFF -set PYARROW_CMAKE_GENERATOR=%GENERATOR% -set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS% -set PYARROW_PARALLEL=2 -set PYARROW_WITH_DATASET=ON -set PYARROW_WITH_FLIGHT=%ARROW_BUILD_FLIGHT% -set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA% -set PYARROW_WITH_PARQUET=ON -set PYARROW_WITH_S3=%ARROW_S3% -set PYARROW_WITH_STATIC_BOOST=ON - -set ARROW_HOME=%CONDA_PREFIX%\Library -@rem ARROW-3075; pkgconfig is broken for Parquet for now -set PARQUET_HOME=%CONDA_PREFIX%\Library - -python setup.py develop -q || exit /B - -set PYTHONDEVMODE=1 - -py.test -r sxX --durations=15 --pyargs pyarrow.tests || exit /B - -@rem -@rem Wheels are built and tested separately (see ARROW-5142). -@rem - diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat deleted file mode 100644 index 616232d20..000000000 --- a/ci/appveyor-cpp-setup.bat +++ /dev/null @@ -1,101 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@echo on - -set "PATH=C:\Miniconda37-x64;C:\Miniconda37-x64\Scripts;C:\Miniconda37-x64\Library\bin;%PATH%" -set BOOST_ROOT=C:\Libraries\boost_1_67_0 -set BOOST_LIBRARYDIR=C:\Libraries\boost_1_67_0\lib64-msvc-14.0 - -@rem -@rem Avoid picking up AppVeyor-installed OpenSSL (linker errors with gRPC) -@rem XXX Perhaps there is a smarter way of solving this issue? -@rem -rd /s /q C:\OpenSSL-Win32 -rd /s /q C:\OpenSSL-Win64 -rd /s /q C:\OpenSSL-v11-Win32 -rd /s /q C:\OpenSSL-v11-Win64 -rd /s /q C:\OpenSSL-v111-Win32 -rd /s /q C:\OpenSSL-v111-Win64 - -@rem -@rem Configure miniconda -@rem -conda config --set auto_update_conda false -conda config --set show_channel_urls True -@rem Help with SSL timeouts to S3 -conda config --set remote_connect_timeout_secs 12 -conda info -a - -@rem -@rem Create conda environment for Build and Toolchain jobs -@rem -@rem Avoid Boost 1.70 because of https://github.com/boostorg/process/issues/85 - -set CONDA_PACKAGES= - -if "%ARROW_BUILD_GANDIVA%" == "ON" ( - @rem Install llvmdev in the toolchain if building gandiva.dll - set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_gandiva_win.yml -) -if "%JOB%" == "Toolchain" ( - @rem Install pre-built "toolchain" packages for faster builds - set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.yml -) -if "%JOB%" NEQ "Build_Debug" ( - @rem Arrow conda environment is only required for the Build and Toolchain jobs - conda create -n arrow -q -y -c conda-forge ^ - --file=ci\conda_env_python.yml ^ - %CONDA_PACKAGES% ^ - "cmake=3.17" ^ - "ninja" ^ - "nomkl" ^ - "pandas" ^ - "fsspec" ^ - "python=%PYTHON%" ^ - || exit /B -) - -@rem -@rem Configure compiler -@rem -if "%GENERATOR%"=="Ninja" set need_vcvarsall=1 -if defined need_vcvarsall ( - @rem Select desired compiler version - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" ( - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64 - ) else ( - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 - ) -) - -@rem -@rem Use clcache for faster builds -@rem -pip install -q git+https://github.com/frerich/clcache.git -@rem Limit cache size to 500 MB -clcache -M 500000000 -clcache -c -clcache -s -powershell.exe -Command "Start-Process clcache-server" - -@rem -@rem Download Minio somewhere on PATH, for unit tests -@rem -if "%ARROW_S3%" == "ON" ( - appveyor DownloadFile https://dl.min.io/server/minio/release/windows-amd64/minio.exe -FileName C:\Windows\Minio.exe || exit /B -) diff --git a/ci/conda_env_archery.yml b/ci/conda_env_archery.yml deleted file mode 100644 index ace7a42ac..000000000 --- a/ci/conda_env_archery.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# cli -click - -# bot, crossbow -github3.py -jinja2 -jira -pygit2 -pygithub -ruamel.yaml -setuptools_scm -toolz - -# benchmark -pandas - -# docker -python-dotenv -#ruamel.yaml - -# release -gitpython -#jinja2 -#jira -semver diff --git a/ci/conda_env_cpp.yml b/ci/conda_env_cpp.yml deleted file mode 100644 index 390eb7dcd..000000000 --- a/ci/conda_env_cpp.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -aws-sdk-cpp -benchmark=1.5.2 -boost-cpp>=1.68.0 -brotli -bzip2 -c-ares -cmake -gflags -glog -gmock>=1.10.0 -grpc-cpp>=1.27.3 -gtest=1.10.0 -libprotobuf -libutf8proc -lz4-c -make -ninja -pkg-config -python -rapidjson -re2 -snappy -thrift-cpp>=0.11.0 -zlib -zstd diff --git a/ci/conda_env_crossbow.txt b/ci/conda_env_crossbow.txt deleted file mode 100644 index 347294650..000000000 --- a/ci/conda_env_crossbow.txt +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -click -github3.py -jinja2 -jira -pygit2 -ruamel.yaml -setuptools_scm -toolz diff --git a/ci/conda_env_gandiva.yml b/ci/conda_env_gandiva.yml deleted file mode 100644 index 024b9fe74..000000000 --- a/ci/conda_env_gandiva.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -clang=11 -llvmdev=11 diff --git a/ci/conda_env_gandiva_win.yml b/ci/conda_env_gandiva_win.yml deleted file mode 100644 index 9098b53d1..000000000 --- a/ci/conda_env_gandiva_win.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# llvmdev=9 or later require Visual Studio 2017 -clangdev=8 -llvmdev=8 diff --git a/ci/conda_env_python.yml b/ci/conda_env_python.yml deleted file mode 100644 index 9124c7e84..000000000 --- a/ci/conda_env_python.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# don't add pandas here, because it is not a mandatory test dependency -boto3 # not a direct dependency of s3fs, but needed for our s3fs fixture -cffi -cython -cloudpickle -fsspec -hypothesis -numpy>=1.16.6 -pytest -pytest-faulthandler -pytest-lazy-fixture -pytz -s3fs>=0.4 -setuptools -setuptools_scm diff --git a/ci/conda_env_r.yml b/ci/conda_env_r.yml deleted file mode 100644 index 03d5f3b62..000000000 --- a/ci/conda_env_r.yml +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -r-assertthat -r-base -r-bit64 -r-dplyr -r-purrr -r-r6 -r-cpp11 -r-rlang -r-tidyselect -r-vctrs -# Test/"Suggests" dependencies -pandoc -r-covr -r-hms -r-lubridate -r-rcmdcheck -r-reticulate -r-rmarkdown -r-testthat -r-tibble diff --git a/ci/conda_env_sphinx.yml b/ci/conda_env_sphinx.yml deleted file mode 100644 index 49388e2b4..000000000 --- a/ci/conda_env_sphinx.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Requirements for building the documentation -breathe -doxygen -ipython -# Pinned per ARROW-9693 -sphinx=3.1.2 -pydata-sphinx-theme diff --git a/ci/conda_env_unix.yml b/ci/conda_env_unix.yml deleted file mode 100644 index 1973238ad..000000000 --- a/ci/conda_env_unix.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# conda package dependencies specific to Unix-like environments (Linux and macOS) - -autoconf -ccache -orc -pkg-config diff --git a/ci/detect-changes.py b/ci/detect-changes.py deleted file mode 100644 index c32f6e040..000000000 --- a/ci/detect-changes.py +++ /dev/null @@ -1,365 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import print_function - -import functools -import os -import pprint -import re -import sys -import subprocess - - -perr = functools.partial(print, file=sys.stderr) - - -def dump_env_vars(prefix, pattern=None): - if pattern is not None: - match = lambda s: re.search(pattern, s) - else: - match = lambda s: True - for name in sorted(os.environ): - if name.startswith(prefix) and match(name): - perr("- {0}: {1!r}".format(name, os.environ[name])) - - -def run_cmd(cmdline): - proc = subprocess.Popen(cmdline, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate() - if proc.returncode != 0: - raise RuntimeError("Command {cmdline} failed with code {returncode}, " - "stderr was:\n{stderr}\n" - .format(cmdline=cmdline, returncode=proc.returncode, - stderr=err.decode())) - return out - - -def get_commit_description(commit): - """ - Return the textual description (title + body) of the given git commit. - """ - out = run_cmd(["git", "show", "--no-patch", "--pretty=format:%B", - commit]) - return out.decode('utf-8', 'ignore') - - -def list_affected_files(commit_range): - """ - Return a list of files changed by the given git commit range. - """ - perr("Getting affected files from", repr(commit_range)) - out = run_cmd(["git", "diff", "--name-only", commit_range]) - return list(filter(None, (s.strip() for s in out.decode().splitlines()))) - - -def get_travis_head_commit(): - return os.environ['TRAVIS_COMMIT'] - - -def get_travis_commit_range(): - if os.environ['TRAVIS_EVENT_TYPE'] == 'pull_request': - # TRAVIS_COMMIT_RANGE is too pessimistic for PRs, as it may contain - # unrelated changes. Instead, use the same strategy as on AppVeyor - # below. - run_cmd(["git", "fetch", "-q", "origin", - "+refs/heads/{0}".format(os.environ['TRAVIS_BRANCH'])]) - merge_base = run_cmd(["git", "merge-base", - "HEAD", "FETCH_HEAD"]).decode().strip() - return "{0}..HEAD".format(merge_base) - else: - cr = os.environ['TRAVIS_COMMIT_RANGE'] - # See - # https://github.com/travis-ci/travis-ci/issues/4596#issuecomment-139811122 - return cr.replace('...', '..') - - -def get_travis_commit_description(): - # Prefer this to get_commit_description(get_travis_head_commit()), - # as rebasing or other repository events may make TRAVIS_COMMIT invalid - # at the time we inspect it - return os.environ['TRAVIS_COMMIT_MESSAGE'] - - -def list_travis_affected_files(): - """ - Return a list of files affected in the current Travis build. - """ - commit_range = get_travis_commit_range() - try: - return list_affected_files(commit_range) - except RuntimeError: - # TRAVIS_COMMIT_RANGE can contain invalid revisions when - # building a branch (not a PR) after rebasing: - # https://github.com/travis-ci/travis-ci/issues/2668 - if os.environ['TRAVIS_EVENT_TYPE'] == 'pull_request': - raise - # If it's a rebase, it's probably enough to use the last commit only - commit_range = '{0}^..'.format(get_travis_head_commit()) - return list_affected_files(commit_range) - - -def list_appveyor_affected_files(): - """ - Return a list of files affected in the current AppVeyor build. - This only works for PR builds. - """ - # Re-fetch PR base branch (e.g. origin/master), pointing FETCH_HEAD to it - run_cmd(["git", "fetch", "-q", "origin", - "+refs/heads/{0}".format(os.environ['APPVEYOR_REPO_BRANCH'])]) - # Compute base changeset between FETCH_HEAD (PR base) and HEAD (PR head) - merge_base = run_cmd(["git", "merge-base", - "HEAD", "FETCH_HEAD"]).decode().strip() - # Compute changes files between base changeset and HEAD - return list_affected_files("{0}..HEAD".format(merge_base)) - - -def list_github_actions_affected_files(): - """ - Return a list of files affected in the current GitHub Actions build. - """ - # GitHub Actions checkout `refs/remotes/pull/$PR/merge` where `HEAD` points - # to the merge commit while `HEAD^` points to the commit before. Hence, - # `..HEAD^` points to all commit between master and the PR. - return list_affected_files("HEAD^..") - - -LANGUAGE_TOPICS = ['c_glib', 'cpp', 'docs', 'go', 'java', 'js', 'python', - 'r', 'ruby', 'rust', 'csharp'] - -ALL_TOPICS = LANGUAGE_TOPICS + ['integration', 'dev'] - - -AFFECTED_DEPENDENCIES = { - 'java': ['integration', 'python'], - 'js': ['integration'], - 'ci': ALL_TOPICS, - 'cpp': ['python', 'c_glib', 'r', 'ruby', 'integration'], - 'format': LANGUAGE_TOPICS, - 'go': ['integration'], - '.travis.yml': ALL_TOPICS, - 'appveyor.yml': ALL_TOPICS, - # In theory, it should ignore CONTRIBUTING.md and ISSUE_TEMPLATE.md, but in - # practice it's going to be CI - '.github': ALL_TOPICS, - 'c_glib': ['ruby'] -} - -COMPONENTS = {'cpp', 'java', 'c_glib', 'r', 'ruby', 'integration', 'js', - 'rust', 'csharp', 'go', 'docs', 'python', 'dev'} - - -def get_affected_topics(affected_files): - """ - Return a dict of topics affected by the given files. - Each dict value is True if affected, False otherwise. - """ - affected = dict.fromkeys(ALL_TOPICS, False) - - for path in affected_files: - parts = [] - head = path - while head: - head, tail = os.path.split(head) - parts.append(tail) - parts.reverse() - assert parts - p = parts[0] - fn = parts[-1] - if fn.startswith('README'): - continue - - if p in COMPONENTS: - affected[p] = True - - _path_already_affected = {} - - def _affect_dependencies(component): - if component in _path_already_affected: - # For circular dependencies, terminate - return - for topic in AFFECTED_DEPENDENCIES.get(component, ()): - affected[topic] = True - _affect_dependencies(topic) - _path_already_affected[topic] = True - - _affect_dependencies(p) - - return affected - - -def make_env_for_topics(affected): - return {'ARROW_CI_{0}_AFFECTED'.format(k.upper()): '1' if v else '0' - for k, v in affected.items()} - - -def get_unix_shell_eval(env): - """ - Return a shell-evalable string to setup some environment variables. - """ - return "; ".join(("export {0}='{1}'".format(k, v) - for k, v in env.items())) - - -def get_windows_shell_eval(env): - """ - Return a shell-evalable string to setup some environment variables. - """ - return "\n".join(('set "{0}={1}"'.format(k, v) - for k, v in env.items())) - - -def run_from_travis(): - perr("Environment variables (excerpt):") - dump_env_vars('TRAVIS_', '(BRANCH|COMMIT|PULL)') - if (os.environ['TRAVIS_REPO_SLUG'] == 'apache/arrow' and - os.environ['TRAVIS_BRANCH'] == 'master' and - os.environ['TRAVIS_EVENT_TYPE'] != 'pull_request'): - # Never skip anything on master builds in the official repository - affected = dict.fromkeys(ALL_TOPICS, True) - else: - desc = get_travis_commit_description() - if '[skip travis]' in desc: - # Skip everything - affected = dict.fromkeys(ALL_TOPICS, False) - elif '[force ci]' in desc or '[force travis]' in desc: - # Test everything - affected = dict.fromkeys(ALL_TOPICS, True) - else: - # Test affected topics - affected_files = list_travis_affected_files() - perr("Affected files:", affected_files) - affected = get_affected_topics(affected_files) - assert set(affected) <= set(ALL_TOPICS), affected - - perr("Affected topics:") - perr(pprint.pformat(affected)) - return get_unix_shell_eval(make_env_for_topics(affected)) - - -def run_from_appveyor(): - perr("Environment variables (excerpt):") - dump_env_vars('APPVEYOR_', '(PULL|REPO)') - if not os.environ.get('APPVEYOR_PULL_REQUEST_HEAD_COMMIT'): - # Not a PR build, test everything - affected = dict.fromkeys(ALL_TOPICS, True) - else: - affected_files = list_appveyor_affected_files() - perr("Affected files:", affected_files) - affected = get_affected_topics(affected_files) - assert set(affected) <= set(ALL_TOPICS), affected - - perr("Affected topics:") - perr(pprint.pformat(affected)) - return get_windows_shell_eval(make_env_for_topics(affected)) - - -def run_from_github(): - perr("Environment variables (excerpt):") - dump_env_vars('GITHUB_', '(REPOSITORY|ACTOR|SHA|REF|HEAD_REF|BASE_REF|EVENT_NAME)') - if os.environ['GITHUB_EVENT_NAME'] != 'pull_request': - # Not a PR build, test everything - affected = dict.fromkeys(ALL_TOPICS, True) - else: - affected_files = list_github_actions_affected_files() - perr("Affected files:", affected_files) - affected = get_affected_topics(affected_files) - assert set(affected) <= set(ALL_TOPICS), affected - - perr("Affected topics:") - perr(pprint.pformat(affected)) - return get_unix_shell_eval(make_env_for_topics(affected)) - - -def test_get_affected_topics(): - affected_topics = get_affected_topics(['cpp/CMakeLists.txt']) - assert affected_topics == { - 'c_glib': True, - 'cpp': True, - 'docs': False, - 'go': False, - 'java': False, - 'js': False, - 'python': True, - 'r': True, - 'ruby': True, - 'rust': False, - 'csharp': False, - 'integration': True, - 'dev': False - } - - affected_topics = get_affected_topics(['format/Schema.fbs']) - assert affected_topics == { - 'c_glib': True, - 'cpp': True, - 'docs': True, - 'go': True, - 'java': True, - 'js': True, - 'python': True, - 'r': True, - 'ruby': True, - 'rust': True, - 'csharp': True, - 'integration': True, - 'dev': False - } - - affected_topics = get_affected_topics(['.github/workflows']) - assert affected_topics == { - 'c_glib': True, - 'cpp': True, - 'docs': True, - 'go': True, - 'java': True, - 'js': True, - 'python': True, - 'r': True, - 'ruby': True, - 'rust': True, - 'csharp': True, - 'integration': True, - 'dev': True, - } - - -if __name__ == "__main__": - # This script should have its output evaluated by a shell, - # e.g. "eval `python ci/detect-changes.py`" - if os.environ.get('TRAVIS'): - try: - print(run_from_travis()) - except Exception: - # Make sure the enclosing eval will return an error - print("exit 1") - raise - elif os.environ.get('APPVEYOR'): - try: - print(run_from_appveyor()) - except Exception: - print("exit 1") - raise - elif os.environ.get('GITHUB_WORKFLOW'): - try: - print(run_from_github()) - except Exception: - print("exit 1") - raise - else: - sys.exit("Script must be run under Travis-CI, AppVeyor or GitHub Actions") diff --git a/ci/docker/conda-cpp.Dockerfile b/ci/docker/conda-cpp.Dockerfile deleted file mode 100644 index 1a5b87ef7..000000000 --- a/ci/docker/conda-cpp.Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch -FROM ${repo}:${arch}-conda - -# install the required conda packages into the test environment -COPY ci/conda_env_cpp.yml \ - ci/conda_env_gandiva.yml \ - /arrow/ci/ -RUN conda install \ - --file arrow/ci/conda_env_cpp.yml \ - --file arrow/ci/conda_env_gandiva.yml \ - compilers \ - doxygen \ - gdb \ - valgrind && \ - conda clean --all - -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DATASET=ON \ - ARROW_DEPENDENCY_SOURCE=CONDA \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA=ON \ - ARROW_HOME=$CONDA_PREFIX \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_S3=ON \ - ARROW_USE_CCACHE=ON \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - PARQUET_BUILD_EXAMPLES=ON \ - PARQUET_BUILD_EXECUTABLES=ON \ - PARQUET_HOME=$CONDA_PREFIX diff --git a/ci/docker/conda-integration.Dockerfile b/ci/docker/conda-integration.Dockerfile deleted file mode 100644 index 1f2c9ac5d..000000000 --- a/ci/docker/conda-integration.Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -FROM ${repo}:${arch}-conda-cpp - -ARG arch=amd64 -ARG maven=3.5 -ARG node=14 -ARG jdk=8 -ARG go=1.15 - -# Install Archery and integration dependencies -COPY ci/conda_env_archery.yml /arrow/ci/ -RUN conda install -q \ - --file arrow/ci/conda_env_cpp.yml \ - --file arrow/ci/conda_env_archery.yml \ - numpy \ - compilers \ - maven=${maven} \ - nodejs=${node} \ - yarn \ - openjdk=${jdk} && \ - conda clean --all --force-pkgs-dirs - -# Install Rust with only the needed components -# (rustfmt is needed for tonic-build to compile the protobuf definitions) -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile=minimal -y && \ - $HOME/.cargo/bin/rustup component add rustfmt - -ENV GOROOT=/opt/go \ - GOBIN=/opt/go/bin \ - GOPATH=/go \ - PATH=/opt/go/bin:$PATH -RUN wget -nv -O - https://dl.google.com/go/go${go}.linux-${arch}.tar.gz | tar -xzf - -C /opt - -ENV ARROW_BUILD_INTEGRATION=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_COMPUTE=OFF \ - ARROW_CSV=OFF \ - ARROW_DATASET=OFF \ - ARROW_FILESYSTEM=OFF \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA=OFF \ - ARROW_HDFS=OFF \ - ARROW_JEMALLOC=OFF \ - ARROW_JSON=OFF \ - ARROW_ORC=OFF \ - ARROW_PARQUET=OFF \ - ARROW_PLASMA=OFF \ - ARROW_S3=OFF \ - ARROW_USE_GLOG=OFF \ - CMAKE_UNITY_BUILD=ON diff --git a/ci/docker/conda-python-dask.Dockerfile b/ci/docker/conda-python-dask.Dockerfile deleted file mode 100644 index cd59a5538..000000000 --- a/ci/docker/conda-python-dask.Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG dask=latest -COPY ci/scripts/install_dask.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_dask.sh ${dask} \ No newline at end of file diff --git a/ci/docker/conda-python-hdfs.Dockerfile b/ci/docker/conda-python-hdfs.Dockerfile deleted file mode 100644 index f6ffc71ce..000000000 --- a/ci/docker/conda-python-hdfs.Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG jdk=8 -ARG maven=3.5 -RUN conda install -q \ - maven=${maven} \ - openjdk=${jdk} \ - pandas && \ - conda clean --all - -# installing libhdfs (JNI) -ARG hdfs=3.2.1 -ENV HADOOP_HOME=/opt/hadoop-${hdfs} \ - HADOOP_OPTS=-Djava.library.path=/opt/hadoop-${hdfs}/lib/native \ - PATH=$PATH:/opt/hadoop-${hdfs}/bin:/opt/hadoop-${hdfs}/sbin -COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/util_download_apache.sh \ - "hadoop/common/hadoop-${hdfs}/hadoop-${hdfs}.tar.gz" /opt - -COPY ci/etc/hdfs-site.xml $HADOOP_HOME/etc/hadoop/ - -# build cpp with tests -ENV CC=gcc \ - CXX=g++ \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=OFF \ - ARROW_PLASMA=OFF \ - ARROW_PARQUET=ON \ - ARROW_ORC=OFF \ - ARROW_HDFS=ON \ - ARROW_PYTHON=ON \ - ARROW_BUILD_TESTS=ON diff --git a/ci/docker/conda-python-jpype.Dockerfile b/ci/docker/conda-python-jpype.Dockerfile deleted file mode 100644 index f77ef9bf6..000000000 --- a/ci/docker/conda-python-jpype.Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG jdk=11 -ARG maven=3.6 -RUN conda install -q \ - maven=${maven} \ - openjdk=${jdk} \ - jpype1 && \ - conda clean --all diff --git a/ci/docker/conda-python-kartothek.Dockerfile b/ci/docker/conda-python-kartothek.Dockerfile deleted file mode 100644 index d52316182..000000000 --- a/ci/docker/conda-python-kartothek.Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -# install kartothek dependencies from conda-forge -RUN conda install -c conda-forge -q \ - attrs \ - click \ - cloudpickle \ - dask \ - decorator \ - freezegun \ - msgpack-python \ - prompt-toolkit \ - pytest-mock \ - pytest-xdist \ - pyyaml \ - simplejson \ - simplekv \ - storefact \ - toolz \ - urlquote \ - zstandard && \ - conda clean --all - -ARG kartothek=latest -COPY ci/scripts/install_kartothek.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_kartothek.sh ${kartothek} /kartothek diff --git a/ci/docker/conda-python-pandas.Dockerfile b/ci/docker/conda-python-pandas.Dockerfile deleted file mode 100644 index 303cc80e4..000000000 --- a/ci/docker/conda-python-pandas.Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG pandas=latest -ARG numpy=latest -COPY ci/scripts/install_pandas.sh /arrow/ci/scripts/ -RUN conda uninstall -q -y numpy && \ - /arrow/ci/scripts/install_pandas.sh ${pandas} ${numpy} diff --git a/ci/docker/conda-python-spark.Dockerfile b/ci/docker/conda-python-spark.Dockerfile deleted file mode 100644 index a2af2ac13..000000000 --- a/ci/docker/conda-python-spark.Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG jdk=8 -ARG maven=3.5 - -RUN conda install -q \ - openjdk=${jdk} \ - maven=${maven} \ - pandas && \ - conda clean --all - -# installing specific version of spark -ARG spark=master -COPY ci/scripts/install_spark.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_spark.sh ${spark} /spark - -# build cpp with tests -ENV CC=gcc \ - CXX=g++ \ - ARROW_PYTHON=ON \ - ARROW_HDFS=ON \ - ARROW_BUILD_TESTS=OFF \ - SPARK_VERSION=${spark} diff --git a/ci/docker/conda-python-turbodbc.Dockerfile b/ci/docker/conda-python-turbodbc.Dockerfile deleted file mode 100644 index ff7fdf6e1..000000000 --- a/ci/docker/conda-python-turbodbc.Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - odbc-postgresql \ - postgresql \ - sudo && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# install turbodbc dependencies from conda-forge -RUN conda install -c conda-forge -q\ - pybind11 \ - pytest-cov \ - mock \ - unixodbc && \ - conda clean --all - -RUN service postgresql start && \ - sudo -u postgres psql -U postgres -c \ - "CREATE DATABASE test_db;" && \ - sudo -u postgres psql -U postgres -c \ - "ALTER USER postgres WITH PASSWORD 'password';" - -ARG turbodbc=latest -COPY ci/scripts/install_turbodbc.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_turbodbc.sh ${turbodbc} /turbodbc - -ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json" diff --git a/ci/docker/conda-python.Dockerfile b/ci/docker/conda-python.Dockerfile deleted file mode 100644 index a7e769748..000000000 --- a/ci/docker/conda-python.Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch -FROM ${repo}:${arch}-conda-cpp - -# install python specific packages -ARG python=3.6 -COPY ci/conda_env_python.yml /arrow/ci/ -RUN conda install -q \ - --file arrow/ci/conda_env_python.yml \ - $([ "$python" == "3.6" -o "$python" == "3.7" ] && echo "pickle5") \ - python=${python} \ - nomkl && \ - conda clean --all - -ENV ARROW_PYTHON=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_TENSORFLOW=ON \ - ARROW_USE_GLOG=OFF diff --git a/ci/docker/conda.Dockerfile b/ci/docker/conda.Dockerfile deleted file mode 100644 index 94de00990..000000000 --- a/ci/docker/conda.Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -FROM ${arch}/ubuntu:18.04 - -# arch is unset after the FROM statement, so need to define it again -ARG arch=amd64 -ARG prefix=/opt/conda - -# install build essentials -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update -y -q && \ - apt-get install -y -q wget tzdata libc6-dbg \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=${prefix}/bin:$PATH -# install conda and minio -COPY ci/scripts/install_conda.sh \ - ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix} -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix} - -# create a conda environment -ADD ci/conda_env_unix.yml /arrow/ci/ -RUN conda create -n arrow --file arrow/ci/conda_env_unix.yml git && \ - conda clean --all - -# activate the created environment by default -RUN echo "conda activate arrow" >> ~/.profile -ENV CONDA_PREFIX=${prefix}/envs/arrow - -# use login shell to activate arrow environment un the RUN commands -SHELL [ "/bin/bash", "-c", "-l" ] - -# use login shell when running the container -ENTRYPOINT [ "/bin/bash", "-c", "-l" ] diff --git a/ci/docker/debian-10-cpp.Dockerfile b/ci/docker/debian-10-cpp.Dockerfile deleted file mode 100644 index 83f8ce529..000000000 --- a/ci/docker/debian-10-cpp.Dockerfile +++ /dev/null @@ -1,104 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -FROM ${arch}/debian:10 -ARG arch - -ENV DEBIAN_FRONTEND noninteractive - -RUN \ - echo "deb http://deb.debian.org/debian buster-backports main" > \ - /etc/apt/sources.list.d/backports.list - -ARG llvm -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ccache \ - clang-${llvm} \ - cmake \ - g++ \ - gcc \ - gdb \ - git \ - libbenchmark-dev \ - libboost-all-dev \ - libbrotli-dev \ - libbz2-dev \ - libc-ares-dev \ - libcurl4-openssl-dev \ - libgflags-dev \ - libgmock-dev \ - libgoogle-glog-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - llvm-${llvm}-dev \ - make \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA=ON \ - ARROW_HOME=/usr/local \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_S3=ON \ - ARROW_USE_CCACHE=ON \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - cares_SOURCE=BUNDLED \ - CC=gcc \ - CXX=g++ \ - gRPC_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PATH=/usr/lib/ccache/:$PATH \ - Protobuf_SOURCE=BUNDLED diff --git a/ci/docker/debian-10-go.Dockerfile b/ci/docker/debian-10-go.Dockerfile deleted file mode 100644 index 199f09e24..000000000 --- a/ci/docker/debian-10-go.Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -ARG go=1.15 -FROM ${arch}/golang:${go} - -# TODO(kszucs): -# 1. add the files required to install the dependencies to .dockerignore -# 2. copy these files to their appropriate path -# 3. download and compile the dependencies diff --git a/ci/docker/debian-10-js.Dockerfile b/ci/docker/debian-10-js.Dockerfile deleted file mode 100644 index 5bb31f2e3..000000000 --- a/ci/docker/debian-10-js.Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -ARG node=14 -FROM ${arch}/node:${node} - -ENV NODE_NO_WARNINGS=1 - -# TODO(kszucs): -# 1. add the files required to install the dependencies to .dockerignore -# 2. copy these files to their appropriate path -# 3. download and compile the dependencies diff --git a/ci/docker/debian-9-java.Dockerfile b/ci/docker/debian-9-java.Dockerfile deleted file mode 100644 index 2cc36e3ea..000000000 --- a/ci/docker/debian-9-java.Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -ARG jdk=8 -ARG maven=3.5.4 -FROM ${arch}/maven:${maven}-jdk-${jdk} - -ENV ARROW_JAVA_SHADE_FLATBUFS=ON - -# TODO(kszucs): -# 1. add the files required to install the dependencies to .dockerignore -# 2. copy these files to their appropriate path -# 3. download and compile the dependencies diff --git a/ci/docker/fedora-33-cpp.Dockerfile b/ci/docker/fedora-33-cpp.Dockerfile deleted file mode 100644 index 9dde69995..000000000 --- a/ci/docker/fedora-33-cpp.Dockerfile +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch -FROM ${arch}/fedora:33 -ARG arch - -# install dependencies -RUN dnf update -y && \ - dnf install -y \ - autoconf \ - boost-devel \ - brotli-devel \ - bzip2-devel \ - c-ares-devel \ - ccache \ - clang-devel \ - cmake \ - curl-devel \ - flatbuffers-devel \ - gcc \ - gcc-c++ \ - gflags-devel \ - git \ - glog-devel \ - gmock-devel \ - google-benchmark-devel \ - grpc-devel \ - grpc-plugins \ - gtest-devel \ - java-latest-openjdk-devel \ - java-latest-openjdk-headless \ - libzstd-devel \ - llvm-devel \ - llvm-static \ - lz4-devel \ - make \ - ninja-build \ - openssl-devel \ - protobuf-devel \ - python \ - rapidjson-devel \ - re2-devel \ - snappy-devel \ - thrift-devel \ - utf8proc-devel \ - wget \ - which \ - zlib-devel - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA_JAVA=ON \ - ARROW_GANDIVA=ON \ - ARROW_HOME=/usr/local \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_S3=ON \ - ARROW_USE_CCACHE=ON \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - CC=gcc \ - CXX=g++ \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXECUTABLES=ON \ - PARQUET_BUILD_EXAMPLES=ON \ - PATH=/usr/lib/ccache/:$PATH diff --git a/ci/docker/linux-apt-c-glib.Dockerfile b/ci/docker/linux-apt-c-glib.Dockerfile deleted file mode 100644 index 12c6e23a0..000000000 --- a/ci/docker/linux-apt-c-glib.Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -RUN apt-get update -y -q && \ - apt-get install -y -q \ - python3 \ - python3-pip \ - gtk-doc-tools \ - libgirepository1.0-dev \ - libglib2.0-doc \ - lsb-release \ - luarocks \ - pkg-config \ - ruby-dev && \ - if [ "$(lsb_release --codename --short)" = "xenial" ]; then \ - apt-get install -y -q --no-install-recommends -t xenial-backports \ - ninja-build; \ - fi && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN luarocks install lgi - -# pip on Ubuntu 20.04 may be buggy: -# -# Collecting meson -# Downloading meson-0.53.2.tar.gz (1.6 MB) -# Installing build dependencies: started -# Installing build dependencies: finished with status 'done' -# Getting requirements to build wheel: started -# Getting requirements to build wheel: finished with status 'error' -# ERROR: Command errored out with exit status 1: -# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay -# cwd: /tmp/pip-install-jn79a_kh/meson -# Complete output (1 lines): -# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py' -# ---------------------------------------- -# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output. -RUN (python3 -m pip install meson || \ - python3 -m pip install --no-use-pep517 meson) && \ - gem install --no-document bundler - -COPY c_glib/Gemfile /arrow/c_glib/ -RUN bundle install --gemfile /arrow/c_glib/Gemfile - -ENV ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_INSTALL_NAME_RPATH=OFF diff --git a/ci/docker/linux-apt-docs.Dockerfile b/ci/docker/linux-apt-docs.Dockerfile deleted file mode 100644 index 20cb889f2..000000000 --- a/ci/docker/linux-apt-docs.Dockerfile +++ /dev/null @@ -1,107 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -ARG r=4.0 -ARG jdk=8 - -# See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu/ -RUN apt-get update -y && \ - apt-get install -y \ - dirmngr \ - apt-transport-https \ - software-properties-common && \ - apt-key adv \ - --keyserver keyserver.ubuntu.com \ - --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \ - apt-get install -y --no-install-recommends \ - autoconf-archive \ - automake \ - curl \ - doxygen \ - gobject-introspection \ - gtk-doc-tools \ - libcurl4-openssl-dev \ - libfontconfig1-dev \ - libfribidi-dev \ - libgirepository1.0-dev \ - libglib2.0-doc \ - libharfbuzz-dev \ - libtiff-dev \ - libtool \ - libxml2-dev \ - ninja-build \ - nvidia-cuda-toolkit \ - openjdk-${jdk}-jdk-headless \ - pandoc \ - r-base=${r}* \ - rsync \ - ruby-dev \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64 - -ARG maven=3.5.4 -COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/util_download_apache.sh \ - "maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" /opt -ENV PATH=/opt/apache-maven-${maven}/bin:$PATH -RUN mvn -version - -ARG node=14 -RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ - apt-get install -y nodejs && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - npm install -g yarn - -RUN pip install \ - meson \ - breathe \ - ipython \ - sphinx \ - pydata-sphinx-theme - -COPY c_glib/Gemfile /arrow/c_glib/ -RUN gem install --no-document bundler && \ - bundle install --gemfile /arrow/c_glib/Gemfile - -# Ensure parallel R package installation, set CRAN repo mirror, -# and use pre-built binaries where possible -COPY ci/etc/rprofile /arrow/ci/etc/ -RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site -# Also ensure parallel compilation of C/C++ code -RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Makeconf - -COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ -COPY r/DESCRIPTION /arrow/r/ -RUN /arrow/ci/scripts/r_deps.sh /arrow && \ - R -e "install.packages('pkgdown')" - -ENV ARROW_FLIGHT=ON \ - ARROW_PYTHON=ON \ - ARROW_S3=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_USE_GLOG=OFF \ - CMAKE_UNITY_BUILD=ON \ diff --git a/ci/docker/linux-apt-jni.Dockerfile b/ci/docker/linux-apt-jni.Dockerfile deleted file mode 100644 index 1abbf05af..000000000 --- a/ci/docker/linux-apt-jni.Dockerfile +++ /dev/null @@ -1,86 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -# pipefail is enabled for proper error detection in the `wget | apt-key add` -# step -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -ENV DEBIAN_FRONTEND noninteractive - -ARG llvm -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - lsb-release \ - software-properties-common \ - wget && \ - code_name=$(lsb_release --codename --short) && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - apt-add-repository -y \ - "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${llvm} main" && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - ca-certificates \ - ccache \ - clang-${llvm} \ - cmake \ - git \ - g++ \ - gcc \ - libboost-all-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - llvm-${llvm}-dev \ - make \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ARG cmake=3.11.4 -RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz | tar -xzf - -C /opt -ENV PATH=/opt/cmake-${cmake}-Linux-x86_64/bin:$PATH - -ENV ARROW_BUILD_TESTS=OFF \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA_JAVA=ON \ - ARROW_GANDIVA=ON \ - ARROW_HOME=/usr/local \ - ARROW_JNI=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA_JAVA_CLIENT=ON \ - ARROW_PLASMA=ON \ - ARROW_USE_CCACHE=ON \ - CC=gcc \ - CXX=g++ \ - ORC_SOURCE=BUNDLED \ - PATH=/usr/lib/ccache/:$PATH \ - Protobuf_SOURCE=BUNDLED diff --git a/ci/docker/linux-apt-lint.Dockerfile b/ci/docker/linux-apt-lint.Dockerfile deleted file mode 100644 index a5c063c74..000000000 --- a/ci/docker/linux-apt-lint.Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM hadolint/hadolint:v1.17.2 AS hadolint -FROM ${base} - -ARG clang_tools -RUN apt-get update && \ - apt-get install -y -q \ - clang-${clang_tools} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - clang-tools-${clang_tools} \ - cmake \ - curl \ - libclang-${clang_tools}-dev \ - llvm-${clang_tools}-dev \ - openjdk-11-jdk-headless \ - python3 \ - python3-dev \ - python3-pip \ - ruby \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Docker linter -COPY --from=hadolint /bin/hadolint /usr/bin/hadolint - -# IWYU -COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ -RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools} - -# Rust linter -ARG rust=nightly-2022-01-17 -RUN curl https://sh.rustup.rs -sSf | \ - sh -s -- --default-toolchain stable -y -ENV PATH /root/.cargo/bin:$PATH -RUN rustup install ${rust} && \ - rustup default ${rust} && \ - rustup component add rustfmt - -# Use python3 by default in scripts -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -COPY dev/archery/requirements.txt \ - dev/archery/requirements-lint.txt \ - /arrow/dev/archery/ -RUN pip install \ - -r arrow/dev/archery/requirements.txt \ - -r arrow/dev/archery/requirements-lint.txt - -ENV LC_ALL=C.UTF-8 \ - LANG=C.UTF-8 diff --git a/ci/docker/linux-apt-python-3.Dockerfile b/ci/docker/linux-apt-python-3.Dockerfile deleted file mode 100644 index 753ba0d3a..000000000 --- a/ci/docker/linux-apt-python-3.Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -RUN apt-get update -y -q && \ - apt-get install -y -q \ - python3 \ - python3-pip \ - python3-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install -U pip setuptools - -COPY python/requirements-build.txt \ - python/requirements-test.txt \ - /arrow/python/ - -RUN pip install \ - -r arrow/python/requirements-build.txt \ - -r arrow/python/requirements-test.txt - -ENV ARROW_PYTHON=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_USE_GLOG=OFF \ diff --git a/ci/docker/linux-apt-r.Dockerfile b/ci/docker/linux-apt-r.Dockerfile deleted file mode 100644 index f47044e33..000000000 --- a/ci/docker/linux-apt-r.Dockerfile +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} -ARG arch - -# Build R -# [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04 -# [2] https://linuxize.com/post/how-to-install-r-on-ubuntu-18-04/#installing-r-packages-from-cran -ARG r=3.6 -RUN apt-get update -y && \ - apt-get install -y \ - dirmngr \ - apt-transport-https \ - software-properties-common && \ - apt-key adv \ - --keyserver keyserver.ubuntu.com \ - --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - # NOTE: R 3.5 and 3.6 are available in the repos with -cran35 suffix - # for trusty, xenial, bionic, and eoan (as of May 2020) - # -cran40 has 4.0 versions for bionic and focal - # R 3.2, 3.3, 3.4 are available without the suffix but only for trusty and xenial - # TODO: make sure OS version and R version are valid together and conditionally set repo suffix - # This is a hack to turn 3.6 into 35 and 4.0 into 40: - add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran'$(echo "${r}" | tr -d . | tr 6 5)'/' && \ - apt-get install -y \ - r-base=${r}* \ - # system libs needed by core R packages - libxml2-dev \ - libgit2-dev \ - libssl-dev \ - # install clang to mirror what was done on Travis - clang \ - clang-format \ - clang-tidy \ - # R CMD CHECK --as-cran needs pdflatex to build the package manual - texlive-latex-base \ - # Need locales so we can set UTF-8 - locales \ - # Need Python to check py-to-r bridge - python3 \ - python3-pip \ - python3-dev && \ - locale-gen en_US.UTF-8 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Ensure parallel R package installation, set CRAN repo mirror, -# and use pre-built binaries where possible -COPY ci/etc/rprofile /arrow/ci/etc/ -RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site -# Also ensure parallel compilation of C/C++ code -RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Makeconf - -COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ -COPY r/DESCRIPTION /arrow/r/ -RUN /arrow/ci/scripts/r_deps.sh /arrow - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -# Set up Python 3 and its dependencies -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -COPY python/requirements-build.txt /arrow/python/ -RUN pip install -r arrow/python/requirements-build.txt - -ENV \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_ORC=OFF \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=OFF \ - ARROW_PYTHON=ON \ - ARROW_S3=ON \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_GLOG=OFF \ - LC_ALL=en_US.UTF-8 diff --git a/ci/docker/linux-apt-ruby.Dockerfile b/ci/docker/linux-apt-ruby.Dockerfile deleted file mode 100644 index 58fd65bd5..000000000 --- a/ci/docker/linux-apt-ruby.Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# depends on a C GLib image -ARG base -FROM ${base} - -COPY ruby/ /arrow/ruby/ -RUN bundle install --gemfile /arrow/ruby/Gemfile -RUN \ - for package in /arrow/ruby/*; do \ - bundle install --gemfile ${package}/Gemfile; \ - done diff --git a/ci/docker/linux-dnf-python-3.Dockerfile b/ci/docker/linux-dnf-python-3.Dockerfile deleted file mode 100644 index 8c3c5c701..000000000 --- a/ci/docker/linux-dnf-python-3.Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -RUN dnf install -y \ - python3 \ - python3-pip \ - python3-devel - -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -COPY python/requirements-build.txt \ - python/requirements-test.txt \ - /arrow/python/ - -RUN pip install \ - -r arrow/python/requirements-build.txt \ - -r arrow/python/requirements-test.txt - -ENV ARROW_PYTHON=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_USE_GLOG=OFF \ diff --git a/ci/docker/linux-r.Dockerfile b/ci/docker/linux-r.Dockerfile deleted file mode 100644 index 8040d3cba..000000000 --- a/ci/docker/linux-r.Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# General purpose Dockerfile to take a Docker image containing R -# and install Arrow R package dependencies - -ARG base -FROM ${base} - -ARG r_bin=R -ENV R_BIN=${r_bin} - -ARG r_dev=FALSE -ENV ARROW_R_DEV=${r_dev} - -ARG devtoolset_version=-1 -ENV DEVTOOLSET_VERSION=${devtoolset_version} - -# Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its Dockerfile) -ENV PATH "${RPREFIX}/bin:${PATH}" - -# Patch up some of the docker images -COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/ -COPY ci/etc/rprofile /arrow/ci/etc/ -COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/r_docker_configure.sh - -COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ -COPY r/DESCRIPTION /arrow/r/ -RUN /arrow/ci/scripts/r_deps.sh /arrow diff --git a/ci/docker/python-sdist.Dockerfile b/ci/docker/python-sdist.Dockerfile deleted file mode 100644 index 853b532ab..000000000 --- a/ci/docker/python-sdist.Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM amd64/ubuntu:20.04 - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - git \ - python3-pip && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -COPY python/requirements-build.txt \ - /arrow/python/requirements-build.txt -RUN pip3 install --requirement /arrow/python/requirements-build.txt - -ENV PYTHON=/usr/bin/python3 diff --git a/ci/docker/python-wheel-manylinux-201x.Dockerfile b/ci/docker/python-wheel-manylinux-201x.Dockerfile deleted file mode 100644 index 19246a467..000000000 --- a/ci/docker/python-wheel-manylinux-201x.Dockerfile +++ /dev/null @@ -1,110 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -ARG arch_alias -ARG arch_short_alias - -RUN yum install -y git flex curl autoconf zip wget - -# Install CMake -ARG cmake=3.19.3 -RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-${arch_alias}.tar.gz -O - | \ - tar -xzf - --directory /usr/local --strip-components=1 - -# Install Ninja -ARG ninja=1.10.2 -RUN mkdir /tmp/ninja && \ - wget -q https://github.com/ninja-build/ninja/archive/v${ninja}.tar.gz -O - | \ - tar -xzf - --directory /tmp/ninja --strip-components=1 && \ - cd /tmp/ninja && \ - ./configure.py --bootstrap && \ - mv ninja /usr/local/bin && \ - rm -rf /tmp/ninja - -# Install ccache -ARG ccache=4.1 -RUN mkdir /tmp/ccache && \ - wget -q https://github.com/ccache/ccache/archive/v${ccache}.tar.gz -O - | \ - tar -xzf - --directory /tmp/ccache --strip-components=1 && \ - cd /tmp/ccache && \ - mkdir build && \ - cd build && \ - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON .. && \ - ninja install && \ - rm -rf /tmp/ccache - -# Install vcpkg -ARG vcpkg -RUN git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \ - git -C /opt/vcpkg checkout ${vcpkg} && \ - /opt/vcpkg/bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics && \ - ln -s /opt/vcpkg/vcpkg /usr/bin/vcpkg - -# Patch ports files as needed -COPY ci/vcpkg arrow/ci/vcpkg -RUN cd /opt/vcpkg && git apply --ignore-whitespace /arrow/ci/vcpkg/ports.patch - -ARG build_type=release -ENV CMAKE_BUILD_TYPE=${build_type} \ - VCPKG_FORCE_SYSTEM_BINARIES=1 \ - VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \ - VCPKG_DEFAULT_TRIPLET=${arch_short_alias}-linux-static-${build_type} \ - VCPKG_FEATURE_FLAGS=-manifests - -# Need to install the boost-build prior installing the boost packages, otherwise -# vcpkg will raise an error. -# TODO(kszucs): factor out the package enumeration to a text file and reuse it -# from the windows image and potentially in a future macos wheel build -RUN vcpkg install --clean-after-build \ - boost-build:${arch_short_alias}-linux && \ - vcpkg install --clean-after-build \ - abseil \ - aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ - boost-filesystem \ - brotli \ - bzip2 \ - c-ares \ - curl \ - flatbuffers \ - gflags \ - glog \ - grpc \ - lz4 \ - openssl \ - orc \ - protobuf \ - rapidjson \ - re2 \ - snappy \ - thrift \ - utf8proc \ - zlib \ - zstd - -ARG python=3.6 -ENV PYTHON_VERSION=${python} -RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \ - echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh - -SHELL ["/bin/bash", "-i", "-c"] -ENTRYPOINT ["/bin/bash", "-i", "-c"] - -COPY python/requirements-wheel-build.txt /arrow/python/ -RUN pip install -r /arrow/python/requirements-wheel-build.txt diff --git a/ci/docker/python-wheel-manylinux-test.Dockerfile b/ci/docker/python-wheel-manylinux-test.Dockerfile deleted file mode 100644 index 55c27d1d7..000000000 --- a/ci/docker/python-wheel-manylinux-test.Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch -ARG python -FROM ${arch}/python:${python} - -# RUN pip install --upgrade pip - -# pandas doesn't provide wheel for aarch64 yet, so cache the compiled -# test dependencies in a docker image -COPY python/requirements-wheel-test.txt /arrow/python/ -RUN pip install -r /arrow/python/requirements-wheel-test.txt diff --git a/ci/docker/python-wheel-windows-vs2017.Dockerfile b/ci/docker/python-wheel-windows-vs2017.Dockerfile deleted file mode 100644 index 0f66a2039..000000000 --- a/ci/docker/python-wheel-windows-vs2017.Dockerfile +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# based on mcr.microsoft.com/windows/servercore:ltsc2019 -# contains choco and vs2017 preinstalled -FROM abrarov/msvc-2017:2.10.0 - -# Install CMake and Ninja -RUN choco install --no-progress -r -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' && \ - choco install --no-progress -r -y gzip wget ninja - -# Add unix tools to path -RUN setx path "%path%;C:\Program Files\Git\usr\bin" - -# Install vcpkg -ARG vcpkg -RUN git clone https://github.com/Microsoft/vcpkg && \ - git -C vcpkg checkout %vcpkg% && \ - vcpkg\bootstrap-vcpkg.bat -disableMetrics -win64 && \ - setx PATH "%PATH%;C:\vcpkg" - -# Patch ports files as needed -COPY ci/vcpkg arrow/ci/vcpkg -RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch - -# Configure vcpkg and install dependencies -# NOTE: use windows batch environment notation for build arguments in RUN -# statements but bash notation in ENV statements -# VCPKG_FORCE_SYSTEM_BINARIES=1 spare around ~750MB of image size if the system -# cmake's and ninja's versions are recent enough -COPY ci/vcpkg arrow/ci/vcpkg -ARG build_type=release -ENV CMAKE_BUILD_TYPE=${build_type} \ - VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \ - VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \ - VCPKG_FEATURE_FLAGS=-manifests -RUN vcpkg install --clean-after-build \ - abseil \ - aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ - boost-filesystem \ - boost-multiprecision \ - boost-system \ - brotli \ - bzip2 \ - c-ares \ - curl \ - flatbuffers \ - gflags \ - glog \ - grpc \ - lz4 \ - openssl \ - orc \ - protobuf \ - rapidjson \ - re2 \ - snappy \ - thrift \ - utf8proc \ - zlib \ - zstd - -# Remove previous installations of python from the base image -RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ - rm -rf Python* - -# Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0) -ARG python=3.6 -RUN (if "%python%"=="3.6" setx PYTHON_VERSION 3.6.8) & \ - (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.4) & \ - (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.6) & \ - (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.1) -RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% -RUN python -m pip install -U pip - -COPY python/requirements-wheel-build.txt arrow/python/ -RUN pip install -r arrow/python/requirements-wheel-build.txt - -# TODO(kszucs): set clcache as the compiler -ENV CLCACHE_DIR="C:\clcache" -RUN pip install clcache - -# For debugging purposes -# RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip -# RUN unzip Dependencies_x64_Release.zip -d Dependencies && setx path "%path%;C:\Depencencies" diff --git a/ci/docker/ubuntu-18.04-cpp.Dockerfile b/ci/docker/ubuntu-18.04-cpp.Dockerfile deleted file mode 100644 index 4b855b526..000000000 --- a/ci/docker/ubuntu-18.04-cpp.Dockerfile +++ /dev/null @@ -1,128 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base=amd64/ubuntu:18.04 -FROM ${base} - -# pipefail is enabled for proper error detection in the `wget | apt-key add` -# step -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -ENV DEBIAN_FRONTEND=noninteractive - -# Installs LLVM toolchain, for Gandiva and testing other compilers -# -# Note that this is installed before the base packages to improve iteration -# while debugging package list with docker build. -ARG clang_tools -ARG llvm -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -ge 10 ]; then \ - echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${clang_tools} main" > \ - /etc/apt/sources.list.d/clang-tools.list; \ - fi && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - clang-${clang_tools} \ - clang-${llvm} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - llvm-${llvm}-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Installs C++ toolchain and dependencies -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ca-certificates \ - ccache \ - cmake \ - g++ \ - gcc \ - gdb \ - git \ - libbenchmark-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libcurl4-openssl-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - liblz4-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libutf8proc-dev \ - libzstd-dev \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - rapidjson-dev \ - tzdata && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Prioritize system packages and local installation -# The following dependencies will be downloaded due to missing/invalid packages -# provided by the distribution: -# - libc-ares-dev does not install CMake config files -# - flatbuffer is not packaged -# - libgtest-dev only provide sources -# - libprotobuf-dev only provide sources -# - thrift is too old -# - s3 tests would require boost-asio that is included since Boost 1.66.0 -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=ON \ - ARROW_HDFS=ON \ - ARROW_HOME=/usr/local \ - ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_USE_ASAN=OFF \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_TSAN=OFF \ - ARROW_USE_UBSAN=OFF \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXECUTABLES=ON \ - PARQUET_BUILD_EXAMPLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ - Thrift_SOURCE=BUNDLED diff --git a/ci/docker/ubuntu-18.04-csharp.Dockerfile b/ci/docker/ubuntu-18.04-csharp.Dockerfile deleted file mode 100644 index 624ce259d..000000000 --- a/ci/docker/ubuntu-18.04-csharp.Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG platform=bionic -ARG dotnet=3.1 -FROM mcr.microsoft.com/dotnet/core/sdk:${dotnet}-${platform} - -RUN dotnet tool install --tool-path /usr/local/bin sourcelink diff --git a/ci/docker/ubuntu-20.04-cpp.Dockerfile b/ci/docker/ubuntu-20.04-cpp.Dockerfile deleted file mode 100644 index 3a37ace13..000000000 --- a/ci/docker/ubuntu-20.04-cpp.Dockerfile +++ /dev/null @@ -1,135 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base=amd64/ubuntu:20.04 -FROM ${base} -ARG arch - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -# Installs LLVM toolchain, for Gandiva and testing other compilers -# -# Note that this is installed before the base packages to improve iteration -# while debugging package list with docker build. -ARG clang_tools -ARG llvm -RUN if [ "${llvm}" -gt "10" ]; then \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ - echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \ - /etc/apt/sources.list.d/clang-tools.list; \ - fi \ - fi && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - clang-${clang_tools} \ - clang-${llvm} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - llvm-${llvm}-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Installs C++ toolchain and dependencies -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ca-certificates \ - ccache \ - cmake \ - g++ \ - gcc \ - gdb \ - git \ - libbenchmark-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libgflags-dev \ - libcurl4-openssl-dev \ - libgoogle-glog-dev \ - liblz4-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - make \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - rapidjson-dev \ - tzdata \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -# Prioritize system packages and local installation -# The following dependencies will be downloaded due to missing/invalid packages -# provided by the distribution: -# - libc-ares-dev does not install CMake config files -# - flatbuffer is not packaged -# - libgtest-dev only provide sources -# - libprotobuf-dev only provide sources -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=ON \ - ARROW_HDFS=ON \ - ARROW_HOME=/usr/local \ - ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_S3=ON \ - ARROW_USE_ASAN=OFF \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_UBSAN=OFF \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXAMPLES=ON \ - PARQUET_BUILD_EXECUTABLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ - PYTHON=python3 diff --git a/ci/docker/ubuntu-20.10-cpp.Dockerfile b/ci/docker/ubuntu-20.10-cpp.Dockerfile deleted file mode 100644 index 80eb072e7..000000000 --- a/ci/docker/ubuntu-20.10-cpp.Dockerfile +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base=amd64/ubuntu:20.10 -FROM ${base} -ARG arch - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -# Installs LLVM toolchain, for Gandiva and testing other compilers -# -# Note that this is installed before the base packages to improve iteration -# while debugging package list with docker build. -ARG clang_tools -ARG llvm -RUN if [ "${llvm}" -gt "10" ]; then \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ - echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${clang_tools} main" > \ - /etc/apt/sources.list.d/clang-tools.list; \ - fi \ - fi && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - clang-${clang_tools} \ - clang-${llvm} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - llvm-${llvm}-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Installs C++ toolchain and dependencies -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ca-certificates \ - ccache \ - cmake \ - g++ \ - gcc \ - gdb \ - git \ - libbenchmark-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libgflags-dev \ - libcurl4-openssl-dev \ - libgoogle-glog-dev \ - libgrpc++-dev \ - liblz4-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - make \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - protobuf-compiler-grpc \ - rapidjson-dev \ - tzdata \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -# Prioritize system packages and local installation -# The following dependencies will be downloaded due to missing/invalid packages -# provided by the distribution: -# - libc-ares-dev does not install CMake config files -# - flatbuffer is not packaged -# - libgtest-dev only provide sources -# - libprotobuf-dev only provide sources -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=ON \ - ARROW_HDFS=ON \ - ARROW_HOME=/usr/local \ - ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_S3=ON \ - ARROW_USE_ASAN=OFF \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_UBSAN=OFF \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXAMPLES=ON \ - PARQUET_BUILD_EXECUTABLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ - PYTHON=python3 diff --git a/ci/etc/hdfs-site.xml b/ci/etc/hdfs-site.xml deleted file mode 100644 index 97214337f..000000000 --- a/ci/etc/hdfs-site.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - dfs.replication - 2 - - - dfs.datanode.data.dir - file:///data/dfs/data - - - dfs.namenode.name.dir - file:///data/dfs/name - - - dfs.namenode.checkpoint.dir - file:///data/dfs/namesecondary - - - dfs.namenode.datanode.registration.ip-hostname-check - false - - - dfs.default.replica - 1 - - - dfs.support.append - true - - - dfs.client.block.write.replace-datanode-on-failure.enable - false - - diff --git a/ci/etc/rprofile b/ci/etc/rprofile deleted file mode 100644 index 229a0101a..000000000 --- a/ci/etc/rprofile +++ /dev/null @@ -1,53 +0,0 @@ - local({ - .pick_cran <- function() { - # Return a CRAN repo URL, preferring RSPM binaries if available for this OS - rspm_template <- "https://packagemanager.rstudio.com/cran/__linux__/%s/latest" - supported_os <- c("focal", "xenial", "bionic", "centos7", "centos8", "opensuse42", "opensuse15", "opensuse152") - - if (nzchar(Sys.which("lsb_release"))) { - os <- tolower(system("lsb_release -cs", intern = TRUE)) - if (os %in% supported_os) { - return(sprintf(rspm_template, os)) - } - } - if (file.exists("/etc/os-release")) { - os_release <- readLines("/etc/os-release") - vals <- sub("^.*=(.*)$", "\\1", os_release) - os <- intersect(vals, supported_os) - if (length(os)) { - # e.g. "bionic" - return(sprintf(rspm_template, os)) - } else { - names(vals) <- sub("^(.*)=.*$", "\\1", os_release) - if (vals["ID"] == "opensuse") { - version <- sub('^"?([0-9]+).*"?.*$', "\\1", vals["VERSION_ID"]) - os <- paste0("opensuse", version) - if (os %in% supported_os) { - return(sprintf(rspm_template, os)) - } - } - } - } - if (file.exists("/etc/system-release")) { - # Something like "CentOS Linux release 7.7.1908 (Core)" - system_release <- tolower(utils::head(readLines("/etc/system-release"), 1)) - # Extract from that the distro and the major version number - os <- sub("^([a-z]+) .* ([0-9]+).*$", "\\1\\2", system_release) - if (os %in% supported_os) { - return(sprintf(rspm_template, os)) - } - } - - return("https://cloud.r-project.org") - } - - options( - Ncpus = parallel::detectCores(), - repos = tryCatch(.pick_cran(), error = function(e) "https://cloud.r-project.org"), - HTTPUserAgent = sprintf( - 'R/%s R (%s)', - getRversion(), - paste(getRversion(), R.version$platform, R.version$arch, R.version$os) - ) - ) -}) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD deleted file mode 100644 index c5b55eef4..000000000 --- a/ci/scripts/PKGBUILD +++ /dev/null @@ -1,134 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -_realname=arrow -pkgbase=mingw-w64-${_realname} -pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}" -pkgver=3.0.0.9000 -pkgrel=8000 -pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)" -arch=("any") -url="https://arrow.apache.org/" -license=("Apache-2.0") -depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" - "${MINGW_PACKAGE_PREFIX}-libutf8proc" - "${MINGW_PACKAGE_PREFIX}-re2" - "${MINGW_PACKAGE_PREFIX}-thrift" - "${MINGW_PACKAGE_PREFIX}-snappy" - "${MINGW_PACKAGE_PREFIX}-zlib" - "${MINGW_PACKAGE_PREFIX}-lz4" - "${MINGW_PACKAGE_PREFIX}-zstd") -makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" - "${MINGW_PACKAGE_PREFIX}-cmake" - "${MINGW_PACKAGE_PREFIX}-gcc") -options=("staticlibs" "strip" "!buildflags") - -# For installing from a local checkout, set source_dir to . and don't include -# a "source" param below -source_dir="$ARROW_HOME" -# else -# source_dir=apache-${_realname}-${pkgver} - -# For released version: -#source=("https://archive.apache.org/dist/arrow/arrow-${pkgver}/apache-arrow-${pkgver}.tar.gz") -#sha256sums=("ac2a77dd9168e9892e432c474611e86ded0be6dfe15f689c948751d37f81391a") -# For github dev version: -# Append `#commit=54b1b2f688e5e84b4c664b1e12a95f93b94ab2f3` to the URL to select a revision -# source=("${source_dir}"::"git+https://github.com/apache/arrow") -# sha256sums=("SKIP") -# source_dir="${APPVEYOR_BUILD_FOLDER}/${source_dir}" - -cpp_build_dir=build-${CARCH}-cpp - -pkgver() { - # The only purpose of this here is to cause the job to error if the - # version in pkgver is different from what is in r/DESCRIPTION - grep Version "${source_dir}/r/DESCRIPTION" | cut -d " " -f 2 -} - -build() { - ARROW_CPP_DIR="${source_dir}/cpp" - [[ -d ${cpp_build_dir} ]] && rm -rf ${cpp_build_dir} - mkdir -p ${cpp_build_dir} - pushd ${cpp_build_dir} - - # The Rtools libutf8proc is a static lib, but Findutf8proc.cmake doesn't - # set the appropriate compiler definition. - export CPPFLAGS="-DUTF8PROC_STATIC" - - # This is the difference between rtools-packages and rtools-backports - # Remove this when submitting to rtools-packages - if [ "$RTOOLS_VERSION" = "35" ]; then - export CC="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin/gcc" - export CXX="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin/g++" - export PATH="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin:$PATH" - export CPPFLAGS="${CPPFLAGS} -I${MINGW_PREFIX}/include" - export LIBS="-L${MINGW_PREFIX}/libs" - export ARROW_S3=OFF - export ARROW_WITH_RE2=OFF - else - export ARROW_S3=ON - export ARROW_WITH_RE2=ON - fi - - MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - ${MINGW_PREFIX}/bin/cmake.exe \ - ${ARROW_CPP_DIR} \ - -G "MSYS Makefiles" \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_STATIC=ON \ - -DARROW_BUILD_UTILITIES=OFF \ - -DARROW_COMPUTE=ON \ - -DARROW_CSV=ON \ - -DARROW_DATASET=ON \ - -DARROW_FILESYSTEM=ON \ - -DARROW_HDFS=OFF \ - -DARROW_JEMALLOC=OFF \ - -DARROW_JSON=ON \ - -DARROW_LZ4_USE_SHARED=OFF \ - -DARROW_MIMALLOC=ON \ - -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ - -DARROW_PARQUET=ON \ - -DARROW_S3="${ARROW_S3}" \ - -DARROW_SNAPPY_USE_SHARED=OFF \ - -DARROW_USE_GLOG=OFF \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_RE2="${ARROW_WITH_RE2}" \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_ZSTD_USE_SHARED=OFF \ - -DARROW_CXXFLAGS="${CPPFLAGS}" \ - -DCMAKE_BUILD_TYPE="release" \ - -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \ - -DCMAKE_UNITY_BUILD=ON \ - -DCMAKE_VERBOSE_MAKEFILE=ON - - make -j3 - popd -} - -package() { - make -C ${cpp_build_dir} DESTDIR="${pkgdir}" install - - local PREFIX_DEPS=$(cygpath -am ${MINGW_PREFIX}) - pushd "${pkgdir}${MINGW_PREFIX}/lib/pkgconfig" - for pc in *.pc; do - sed -s "s|${PREFIX_DEPS}|${MINGW_PREFIX}|g" -i $pc - done - popd -} diff --git a/ci/scripts/c_glib_build.sh b/ci/scripts/c_glib_build.sh deleted file mode 100755 index ce3cea18e..000000000 --- a/ci/scripts/c_glib_build.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/c_glib -build_dir=${2}/c_glib -: ${ARROW_GLIB_GTK_DOC:=false} -: ${ARROW_GLIB_DEVELOPMENT_MODE:=false} - -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig - -export CFLAGS="-DARROW_NO_DEPRECATED_API" -export CXXFLAGS="-DARROW_NO_DEPRECATED_API" - -mkdir -p ${build_dir} - -# Build with Meson -meson --prefix=$ARROW_HOME \ - --libdir=lib \ - -Ddevelopment_mode=${ARROW_GLIB_DEVELOPMENT_MODE} \ - -Dgtk_doc=${ARROW_GLIB_GTK_DOC} \ - ${build_dir} \ - ${source_dir} - -pushd ${build_dir} -ninja -ninja install -popd diff --git a/ci/scripts/c_glib_test.sh b/ci/scripts/c_glib_test.sh deleted file mode 100755 index 25c54138e..000000000 --- a/ci/scripts/c_glib_test.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/c_glib -build_dir=${2}/c_glib - -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig -export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 - -pushd ${source_dir} - -ruby test/run-test.rb - -if [[ "$(uname -s)" == "Linux" ]]; then - # TODO(kszucs): on osx it fails to load 'lgi.corelgilua51' despite that lgi - # was installed by luarocks - pushd example/lua - lua write-batch.lua - lua read-batch.lua - lua write-stream.lua - lua read-stream.lua - popd -fi - -popd - -pushd ${build_dir} -example/extension-type -popd diff --git a/ci/scripts/ccache_setup.sh b/ci/scripts/ccache_setup.sh deleted file mode 100755 index f77fbb373..000000000 --- a/ci/scripts/ccache_setup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -echo "ARROW_USE_CCACHE=ON" >> $GITHUB_ENV -echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV -echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV -echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV -echo "CCACHE_MAXSIZE=500M" >> $GITHUB_ENV diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh deleted file mode 100755 index 8a1e4f32f..000000000 --- a/ci/scripts/cpp_build.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/cpp -build_dir=${2}/cpp -with_docs=${3:-false} - -: ${ARROW_USE_CCACHE:=OFF} - -# TODO(kszucs): consider to move these to CMake -if [ ! -z "${CONDA_PREFIX}" ]; then - echo -e "===\n=== Conda environment for build\n===" - conda list - - export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_AR=${AR} -DCMAKE_RANLIB=${RANLIB}" - export ARROW_GANDIVA_PC_CXX_FLAGS=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') -elif [ -x "$(command -v xcrun)" ]; then - export ARROW_GANDIVA_PC_CXX_FLAGS="-isysroot;$(xcrun --show-sdk-path)" -fi - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo -e "===\n=== ccache statistics before build\n===" - ccache -s -fi - -mkdir -p ${build_dir} -pushd ${build_dir} - -cmake -G "${CMAKE_GENERATOR:-Ninja}" \ - -DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \ - -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \ - -DARROW_BUILD_BENCHMARKS_REFERENCE=${ARROW_BUILD_BENCHMARKS:-OFF} \ - -DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-OFF} \ - -DARROW_BUILD_INTEGRATION=${ARROW_BUILD_INTEGRATION:-OFF} \ - -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \ - -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \ - -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \ - -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ - -DARROW_COMPUTE=${ARROW_COMPUTE:-ON} \ - -DARROW_CSV=${ARROW_CSV:-ON} \ - -DARROW_CUDA=${ARROW_CUDA:-OFF} \ - -DARROW_CXXFLAGS=${ARROW_CXXFLAGS:-} \ - -DARROW_DATASET=${ARROW_DATASET:-ON} \ - -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ - -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \ - -DARROW_ENABLE_TIMING_TESTS=${ARROW_ENABLE_TIMING_TESTS:-ON} \ - -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \ - -DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \ - -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \ - -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ - -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \ - -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ - -DARROW_HDFS=${ARROW_HDFS:-ON} \ - -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \ - -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ - -DARROW_JNI=${ARROW_JNI:-OFF} \ - -DARROW_JSON=${ARROW_JSON:-ON} \ - -DARROW_LARGE_MEMORY_TESTS=${ARROW_LARGE_MEMORY_TESTS:-OFF} \ - -DARROW_MIMALLOC=${ARROW_MIMALLOC:-OFF} \ - -DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \ - -DARROW_ORC=${ARROW_ORC:-OFF} \ - -DARROW_PARQUET=${ARROW_PARQUET:-OFF} \ - -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT:-OFF} \ - -DARROW_PLASMA=${ARROW_PLASMA:-OFF} \ - -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ - -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \ - -DARROW_S3=${ARROW_S3:-OFF} \ - -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ - -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \ - -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ - -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-ON} \ - -DARROW_USE_GLOG=${ARROW_USE_GLOG:-OFF} \ - -DARROW_USE_LD_GOLD=${ARROW_USE_LD_GOLD:-OFF} \ - -DARROW_USE_PRECOMPILED_HEADERS=${ARROW_USE_PRECOMPILED_HEADERS:-OFF} \ - -DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \ - -DARROW_USE_TSAN=${ARROW_USE_TSAN:-OFF} \ - -DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \ - -DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \ - -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-OFF} \ - -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-OFF} \ - -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-OFF} \ - -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-OFF} \ - -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \ - -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ - -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ - -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ - -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ - -DBOOST_SOURCE=${BOOST_SOURCE:-} \ - -DBrotli_SOURCE=${Brotli_SOURCE:-} \ - -DBUILD_WARNING_LEVEL=${BUILD_WARNING_LEVEL:-CHECKIN} \ - -Dc-ares_SOURCE=${cares_SOURCE:-} \ - -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ - -DCMAKE_C_FLAGS="${CFLAGS:-}" \ - -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \ - -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ - -Dgflags_SOURCE=${gflags_SOURCE:-} \ - -DgRPC_SOURCE=${gRPC_SOURCE:-} \ - -DGTest_SOURCE=${GTest_SOURCE:-} \ - -DLz4_SOURCE=${Lz4_SOURCE:-} \ - -DORC_SOURCE=${ORC_SOURCE:-} \ - -DPARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-OFF} \ - -DPARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-OFF} \ - -DPARQUET_REQUIRE_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON} \ - -DProtobuf_SOURCE=${Protobuf_SOURCE:-} \ - -DRapidJSON_SOURCE=${RapidJSON_SOURCE:-} \ - -Dre2_SOURCE=${re2_SOURCE:-} \ - -DSnappy_SOURCE=${Snappy_SOURCE:-} \ - -DThrift_SOURCE=${Thrift_SOURCE:-} \ - -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \ - -Dzstd_SOURCE=${zstd_SOURCE:-} \ - ${CMAKE_ARGS} \ - ${source_dir} - -if [ ! -z "${CPP_MAKE_PARALLELISM}" ]; then - time cmake --build . --target install -- -j${CPP_MAKE_PARALLELISM} -else - time cmake --build . --target install -fi - -popd - -if [ -x "$(command -v ldconfig)" ]; then - ldconfig -fi - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo -e "===\n=== ccache statistics after build\n===" - ccache -s -fi - -if [ "${with_docs}" == "true" ]; then - pushd ${source_dir}/apidoc - doxygen - popd -fi diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh deleted file mode 100755 index 1bf0a3b88..000000000 --- a/ci/scripts/cpp_test.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/cpp -build_dir=${2}/cpp -binary_output_dir=${build_dir}/${ARROW_BUILD_TYPE:-debug} - -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data -export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${LD_LIBRARY_PATH} - -# By default, aws-sdk tries to contact a non-existing local ip host -# to retrieve metadata. Disable this so that S3FileSystem tests run faster. -export AWS_EC2_METADATA_DISABLED=TRUE - -ctest_options=() -case "$(uname)" in - Linux) - n_jobs=$(nproc) - ;; - Darwin) - n_jobs=$(sysctl -n hw.ncpu) - ;; - MINGW*) - n_jobs=${NUMBER_OF_PROCESSORS:-1} - # TODO: Enable these crashed tests. - # https://issues.apache.org/jira/browse/ARROW-9072 - exclude_tests="gandiva-internals-test" - exclude_tests="${exclude_tests}|gandiva-projector-test" - exclude_tests="${exclude_tests}|gandiva-utf8-test" - if [ "${MSYSTEM}" = "MINGW32" ]; then - exclude_tests="${exclude_tests}|gandiva-projector-test" - exclude_tests="${exclude_tests}|gandiva-binary-test" - exclude_tests="${exclude_tests}|gandiva-boolean-expr-test" - exclude_tests="${exclude_tests}|gandiva-date-time-test" - exclude_tests="${exclude_tests}|gandiva-decimal-single-test" - exclude_tests="${exclude_tests}|gandiva-decimal-test" - exclude_tests="${exclude_tests}|gandiva-filter-project-test" - exclude_tests="${exclude_tests}|gandiva-filter-test" - exclude_tests="${exclude_tests}|gandiva-hash-test" - exclude_tests="${exclude_tests}|gandiva-if-expr-test" - exclude_tests="${exclude_tests}|gandiva-in-expr-test" - exclude_tests="${exclude_tests}|gandiva-literal-test" - exclude_tests="${exclude_tests}|gandiva-null-validity-test" - fi - ctest_options+=(--exclude-regex "${exclude_tests}") - ;; - *) - n_jobs=${NPROC:-1} - ;; -esac - -pushd ${build_dir} - -if ! which python > /dev/null 2>&1; then - export PYTHON=python3 -fi -ctest \ - --label-regex unittest \ - --output-on-failure \ - --parallel ${n_jobs} \ - --timeout 300 \ - "${ctest_options[@]}" - -if [ "${ARROW_FUZZING}" == "ON" ]; then - # Fuzzing regression tests - ${binary_output_dir}/arrow-ipc-stream-fuzz ${ARROW_TEST_DATA}/arrow-ipc-stream/crash-* - ${binary_output_dir}/arrow-ipc-stream-fuzz ${ARROW_TEST_DATA}/arrow-ipc-stream/*-testcase-* - ${binary_output_dir}/arrow-ipc-file-fuzz ${ARROW_TEST_DATA}/arrow-ipc-file/*-testcase-* - ${binary_output_dir}/arrow-ipc-tensor-stream-fuzz ${ARROW_TEST_DATA}/arrow-ipc-tensor-stream/*-testcase-* - if [ "${ARROW_PARQUET}" == "ON" ]; then - ${binary_output_dir}/parquet-arrow-fuzz ${ARROW_TEST_DATA}/parquet/fuzzing/*-testcase-* - fi -fi - -popd diff --git a/ci/scripts/csharp_build.sh b/ci/scripts/csharp_build.sh deleted file mode 100755 index 5a3976794..000000000 --- a/ci/scripts/csharp_build.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/csharp - -pushd ${source_dir} -dotnet build -popd diff --git a/ci/scripts/csharp_pack.sh b/ci/scripts/csharp_pack.sh deleted file mode 100755 index e9dfc664e..000000000 --- a/ci/scripts/csharp_pack.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -source_dir=${1}/csharp - -pushd ${source_dir} -dotnet pack -c Release -popd diff --git a/ci/scripts/csharp_test.sh b/ci/scripts/csharp_test.sh deleted file mode 100755 index 9e4e35dd4..000000000 --- a/ci/scripts/csharp_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/csharp - -pushd ${source_dir} -dotnet test -for pdb in artifacts/Apache.Arrow/*/*/Apache.Arrow.pdb; do - sourcelink test ${pdb} -done -popd diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh deleted file mode 100755 index e6ee768ee..000000000 --- a/ci/scripts/docs_build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -ex - -arrow_dir=${1} -build_dir=${2}/docs - -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig:${PKG_CONFIG_PATH} -export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 -export CFLAGS="-DARROW_NO_DEPRECATED_API" -export CXXFLAGS="-DARROW_NO_DEPRECATED_API" - -ncpus=$(python3 -c "import os; print(os.cpu_count())") - -# Sphinx docs -sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir} - -# C++ - original doxygen -# rsync -a ${arrow_dir}/cpp/apidoc/ ${build_dir}/cpp - -# R -rsync -a ${arrow_dir}/r/docs/ ${build_dir}/r - -# C GLib -rsync -a ${ARROW_HOME}/share/gtk-doc/html/ ${build_dir}/c_glib - -# Java -rsync -a ${arrow_dir}/java/target/site/apidocs/ ${build_dir}/java/reference - -# Javascript -rsync -a ${arrow_dir}/js/doc/ ${build_dir}/js diff --git a/ci/scripts/go_build.sh b/ci/scripts/go_build.sh deleted file mode 100755 index 7093be4d2..000000000 --- a/ci/scripts/go_build.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/go - -pushd ${source_dir}/arrow - -go get -d -t -v ./... -go install -v ./... - -popd - -pushd ${source_dir}/parquet - -go get -d -t -v ./... -go install -v ./... - -popd diff --git a/ci/scripts/go_test.sh b/ci/scripts/go_test.sh deleted file mode 100755 index 7dd873df3..000000000 --- a/ci/scripts/go_test.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/go - -pushd ${source_dir}/arrow - -for d in $(go list ./... | grep -v vendor); do - go test $d -done - -popd - -pushd ${source_dir}/parquet - -for d in $(go list ./... | grep -v vendor); do - go test $d -done - -popd diff --git a/ci/scripts/install_conda.sh b/ci/scripts/install_conda.sh deleted file mode 100755 index f4d313b63..000000000 --- a/ci/scripts/install_conda.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -declare -A archs -archs=([amd64]=x86_64 - [arm32v7]=armv7l - [ppc64le]=ppc64le - [i386]=x86) - -declare -A platforms -platforms=([windows]=Windows - [macos]=MacOSX - [linux]=Linux) - -if [ "$#" -ne 4 ]; then - echo "Usage: $0 " - exit 1 -elif [[ -z ${archs[$1]} ]]; then - echo "Unexpected architecture: ${1}" - exit 1 -elif [[ -z ${platforms[$2]} ]]; then - echo "Unexpected platform: ${2}" - exit 1 -fi - -arch=${archs[$1]} -platform=${platforms[$2]} -version=$3 -prefix=$4 - -echo "Downloading Miniconda installer..." -wget -nv https://repo.continuum.io/miniconda/Miniconda3-${version}-${platform}-${arch}.sh -O /tmp/miniconda.sh -bash /tmp/miniconda.sh -b -p ${prefix} -rm /tmp/miniconda.sh - -# Like "conda init", but for POSIX sh rather than bash -ln -s ${prefix}/etc/profile.d/conda.sh /etc/profile.d/conda.sh - -# Configure -source /etc/profile.d/conda.sh -conda config --add channels conda-forge -conda config --set channel_priority strict -conda config --set show_channel_urls True -conda config --set remote_connect_timeout_secs 12 - -# Update and clean -conda update --all -y -conda clean --all -y diff --git a/ci/scripts/install_dask.sh b/ci/scripts/install_dask.sh deleted file mode 100755 index 954ce3249..000000000 --- a/ci/scripts/install_dask.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -dask=$1 - -if [ "${dask}" = "master" ]; then - pip install https://github.com/dask/dask/archive/main.tar.gz#egg=dask[dataframe] -elif [ "${dask}" = "latest" ]; then - conda install -q dask -else - conda install -q dask=${dask} -fi -conda clean --all diff --git a/ci/scripts/install_iwyu.sh b/ci/scripts/install_iwyu.sh deleted file mode 100755 index 3cd2cbc95..000000000 --- a/ci/scripts/install_iwyu.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -eu - -source_dir=${1:-/tmp/iwyu} -install_prefix=${2:-/usr/local} -clang_tools_version=${3:-8} - -iwyu_branch_name="clang_${clang_tools_version}" -if [ ${clang_tools_version} -lt 10 ]; then - iwyu_branch_name="${iwyu_branch_name}.0" -fi - -git clone --single-branch --branch ${iwyu_branch_name} \ - https://github.com/include-what-you-use/include-what-you-use.git ${source_dir} - -mkdir -p ${source_dir}/build -pushd ${source_dir}/build - -# Build IWYU for current Clang -export CC=clang-${clang_tools_version} -export CXX=clang++-${clang_tools_version} - -cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${clang_tools_version} \ - -DCMAKE_INSTALL_PREFIX=${install_prefix} \ - ${source_dir} -make -j4 -make install - -popd - -rm -rf ${source_dir} diff --git a/ci/scripts/install_kartothek.sh b/ci/scripts/install_kartothek.sh deleted file mode 100755 index 4d88943b6..000000000 --- a/ci/scripts/install_kartothek.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -karthothek=$1 -target=$2 - -git clone --recurse-submodules https://github.com/JDASoftwareGroup/kartothek "${target}" -if [ "${kartothek}" = "master" ]; then - git -C "${target}" checkout master; -elif [ "${kartothek}" = "latest" ]; then - git -C "${target}" checkout $(git describe --tags); -else - git -C "${target}" checkout ${kartothek}; -fi - -pushd "${target}" -pip install --no-deps . -popd diff --git a/ci/scripts/install_minio.sh b/ci/scripts/install_minio.sh deleted file mode 100755 index 42f7ce040..000000000 --- a/ci/scripts/install_minio.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -declare -A archs -archs=([amd64]=amd64 - [arm64v8]=arm64 - [arm32v7]=arm - [s390x]=s390x) - -declare -A platforms -platforms=([linux]=linux - [macos]=darwin) - -arch=${archs[$1]} -platform=${platforms[$2]} -version=$3 -prefix=$4 - -if [ "$#" -ne 4 ]; then - echo "Usage: $0 " - exit 1 -elif [[ -z ${arch} ]]; then - echo "Unexpected architecture: ${1}" - exit 1 -elif [[ -z ${platform} ]]; then - echo "Unexpected platform: ${2}" - exit 1 -elif [[ ${version} != "latest" ]]; then - echo "Cannot fetch specific versions of minio, only latest is supported." - exit 1 -fi - -wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/${platform}-${arch}/minio -chmod +x ${prefix}/bin/minio diff --git a/ci/scripts/install_osx_sdk.sh b/ci/scripts/install_osx_sdk.sh deleted file mode 100755 index 896d084e0..000000000 --- a/ci/scripts/install_osx_sdk.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -if [ ${using_homebrew} != "yes" ]; then - export MACOSX_DEPLOYMENT_TARGET="10.9" - export CONDA_BUILD_SYSROOT="$(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk" - - if [[ ! -d ${CONDA_BUILD_SYSROOT} || "$OSX_FORCE_SDK_DOWNLOAD" == "1" ]]; then - echo "downloading ${macosx_deployment_target} sdk" - curl -L -O https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz - tar -xf MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz -C "$(dirname "$CONDA_BUILD_SYSROOT")" - # set minimum sdk version to our target - plutil -replace MinimumSDKVersion -string ${MACOSX_DEPLOYMENT_TARGET} $(xcode-select -p)/Platforms/MacOSX.platform/Info.plist - plutil -replace DTSDKName -string macosx${MACOSX_DEPLOYMENT_TARGET}internal $(xcode-select -p)/Platforms/MacOSX.platform/Info.plist - fi - - if [ -d "${CONDA_BUILD_SYSROOT}" ]; then - echo "Found CONDA_BUILD_SYSROOT: ${CONDA_BUILD_SYSROOT}" - else - echo "Missing CONDA_BUILD_SYSROOT: ${CONDA_BUILD_SYSROOT}" - exit 1 - fi -fi diff --git a/ci/scripts/install_pandas.sh b/ci/scripts/install_pandas.sh deleted file mode 100755 index 5aca65f82..000000000 --- a/ci/scripts/install_pandas.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -pandas=$1 -numpy=${2:-"latest"} - -if [ "${numpy}" = "nightly" ]; then - pip install --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple --pre numpy -elif [ "${numpy}" = "latest" ]; then - pip install numpy -else - pip install numpy==${numpy} -fi - -if [ "${pandas}" = "master" ]; then - pip install git+https://github.com/pandas-dev/pandas.git --no-build-isolation -elif [ "${pandas}" = "nightly" ]; then - pip install --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple --pre pandas -elif [ "${pandas}" = "latest" ]; then - pip install pandas -else - pip install pandas==${pandas} -fi diff --git a/ci/scripts/install_spark.sh b/ci/scripts/install_spark.sh deleted file mode 100755 index 936313fd8..000000000 --- a/ci/scripts/install_spark.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -spark=$1 -target=$2 - -git clone https://github.com/apache/spark "${target}" -git -C "${target}" checkout "${spark}" diff --git a/ci/scripts/install_turbodbc.sh b/ci/scripts/install_turbodbc.sh deleted file mode 100755 index a71520beb..000000000 --- a/ci/scripts/install_turbodbc.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -turbodbc=$1 -target=$2 - -git clone --recurse-submodules https://github.com/blue-yonder/turbodbc "${target}" -if [ "${turbodbc}" = "master" ]; then - git -C "${target}" checkout master; -elif [ "${turbodbc}" = "latest" ]; then - git -C "${target}" checkout $(git describe --tags); -else - git -C "${target}" checkout ${turbodbc}; -fi diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh deleted file mode 100755 index 5d2e71916..000000000 --- a/ci/scripts/integration_arrow.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/cpp -build_dir=${2}/cpp - -gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration - -pip install -e $arrow_dir/dev/archery - -archery integration --with-all --run-flight \ - --gold-dirs=$gold_dir/0.14.1 \ - --gold-dirs=$gold_dir/0.17.1 \ - --gold-dirs=$gold_dir/1.0.0-bigendian \ - --gold-dirs=$gold_dir/1.0.0-littleendian \ - --gold-dirs=$gold_dir/2.0.0-compression \ - --gold-dirs=$gold_dir/4.0.0-shareddict \ diff --git a/ci/scripts/integration_dask.sh b/ci/scripts/integration_dask.sh deleted file mode 100755 index a73592877..000000000 --- a/ci/scripts/integration_dask.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -# check that optional pyarrow modules are available -# because pytest would just skip the dask tests -python -c "import pyarrow.orc" -python -c "import pyarrow.parquet" - -# check that dask.dataframe is correctly installed -python -c "import dask.dataframe" - -# TODO(kszucs): the following tests are also uses pyarrow -# pytest -sv --pyargs dask.bytes.tests.test_s3 -# pytest -sv --pyargs dask.bytes.tests.test_hdfs -# pytest -sv --pyargs dask.bytes.tests.test_local - -# skip failing pickle test, see https://github.com/dask/dask/issues/6374 -pytest -v --pyargs dask.dataframe.tests.test_dataframe -k "not test_dataframe_picklable" -pytest -v --pyargs dask.dataframe.io.tests.test_orc -# skip failing parquet tests, see https://github.com/dask/dask/issues/6243 -# test_illegal_column_name can be removed once next dask release is out -# (https://github.com/dask/dask/pull/6378) -pytest -v --pyargs dask.dataframe.io.tests.test_parquet \ - -k "not test_to_parquet_pyarrow_w_inconsistent_schema_by_partition_fails_by_default and not test_timeseries_nulls_in_schema and not test_illegal_column_name" diff --git a/ci/scripts/integration_hdfs.sh b/ci/scripts/integration_hdfs.sh deleted file mode 100755 index c95449379..000000000 --- a/ci/scripts/integration_hdfs.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source_dir=${1}/cpp -build_dir=${2}/cpp - -export CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath --glob) -export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop -export LIBHDFS3_CONF=$HADOOP_CONF_DIR/hdfs-site.xml -export ARROW_LIBHDFS3_DIR=$CONDA_PREFIX/lib - -libhdfs_dir=$HADOOP_HOME/lib/native -hadoop_home=$HADOOP_HOME - -function use_hadoop_home() { - unset ARROW_LIBHDFS_DIR - export HADOOP_HOME=$hadoop_home -} - -function use_libhdfs_dir() { - unset HADOOP_HOME - export ARROW_LIBHDFS_DIR=$libhdfs_dir -} - -# execute cpp tests -export ARROW_HDFS_TEST_LIBHDFS_REQUIRE=ON -pushd ${build_dir} - -debug/arrow-io-hdfs-test -debug/arrow-hdfs-test - -use_libhdfs_dir -debug/arrow-io-hdfs-test -debug/arrow-hdfs-test -use_hadoop_home - -popd - -# cannot use --pyargs with custom arguments like --hdfs or --only-hdfs, because -# pytest ignores them, see https://github.com/pytest-dev/pytest/issues/3517 -export PYARROW_TEST_HDFS=ON - -export PYARROW_HDFS_TEST_LIBHDFS_REQUIRE=ON - -pytest -vs --pyargs pyarrow.tests.test_fs -pytest -vs --pyargs pyarrow.tests.test_hdfs - -use_libhdfs_dir -pytest -vs --pyargs pyarrow.tests.test_fs -pytest -vs --pyargs pyarrow.tests.test_hdfs -use_hadoop_home diff --git a/ci/scripts/integration_hiveserver2.sh b/ci/scripts/integration_hiveserver2.sh deleted file mode 100755 index 36fba5ca8..000000000 --- a/ci/scripts/integration_hiveserver2.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -arrow_dir=${1} -source_dir=${1}/cpp -build_dir=${2}/cpp - -${arrow_dir}/ci/scripts/util_wait_for_it.sh impala:21050 -t 300 -s -- echo "impala is up" - -pushd ${build_dir} - -# ninja hiveserver2-test -debug/hiveserver2-test - -popd diff --git a/ci/scripts/integration_kartothek.sh b/ci/scripts/integration_kartothek.sh deleted file mode 100755 index 379569b9c..000000000 --- a/ci/scripts/integration_kartothek.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -# check that optional pyarrow modules are available -# because pytest would just skip the pyarrow tests -python -c "import pyarrow.parquet" - -# check that kartothek is correctly installed -python -c "import kartothek" - -pushd /kartothek -# See ARROW-12314, test_load_dataframes_columns_raises_missing skipped because of changed error message -pytest -n0 --ignore tests/cli/test_query.py -k "not test_load_dataframes_columns_raises_missing" diff --git a/ci/scripts/integration_spark.sh b/ci/scripts/integration_spark.sh deleted file mode 100755 index a53a62971..000000000 --- a/ci/scripts/integration_spark.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# exit on any error -set -eu - -source_dir=${1} -spark_dir=${2} - -# Test Spark with latest PyArrow only, don't build with latest Arrow Java -test_pyarrow_only=${3:-false} - -# Spark branch to checkout -spark_version=${SPARK_VERSION:-master} - -# Use old behavior that always dropped tiemzones. -export PYARROW_IGNORE_TIMEZONE=1 - -if [ "${SPARK_VERSION:0:2}" == "2." ]; then - # https://github.com/apache/spark/blob/master/docs/sql-pyspark-pandas-with-arrow.md#compatibility-setting-for-pyarrow--0150-and-spark-23x-24x - export ARROW_PRE_0_15_IPC_FORMAT=1 -fi - -# Get Arrow Java version -pushd ${source_dir}/java - arrow_version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | sed -n -e '/^\[.*\]/ !{ /^[0-9]/ { p; q } }'` -popd - -export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=warn" -export MAVEN_OPTS="${MAVEN_OPTS} -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" - -pushd ${spark_dir} - - if [ "${test_pyarrow_only}" == "true" ]; then - echo "Building Spark ${SPARK_VERSION} to test pyarrow only" - - # Build Spark only - build/mvn -B -DskipTests package - - else - - # Update Spark pom with the Arrow version just installed and build Spark, need package phase for pyspark - echo "Building Spark ${SPARK_VERSION} with Arrow ${arrow_version}" - mvn versions:set-property -Dproperty=arrow.version -DnewVersion=${arrow_version} - - # Build Spark with new Arrow Java - build/mvn -B -DskipTests package -pl sql/core -pl assembly -am - - spark_scala_tests=( - "org.apache.spark.sql.execution.arrow" - "org.apache.spark.sql.execution.vectorized.ColumnarBatchSuite" - "org.apache.spark.sql.execution.vectorized.ArrowColumnVectorSuite") - - (echo "Testing Spark:"; IFS=$'\n'; echo "${spark_scala_tests[*]}") - - # TODO: should be able to only build spark-sql tests with adding "-pl sql/core" but not currently working - build/mvn -B -Dtest=none -DwildcardSuites=$(IFS=,; echo "${spark_scala_tests[*]}") test - fi - - # Run pyarrow related Python tests only - spark_python_tests=( - "pyspark.sql.tests.test_arrow" - "pyspark.sql.tests.test_pandas_map" - "pyspark.sql.tests.test_pandas_cogrouped_map" - "pyspark.sql.tests.test_pandas_grouped_map" - "pyspark.sql.tests.test_pandas_udf" - "pyspark.sql.tests.test_pandas_udf_scalar" - "pyspark.sql.tests.test_pandas_udf_grouped_agg" - "pyspark.sql.tests.test_pandas_udf_window") - - (echo "Testing PySpark:"; IFS=$'\n'; echo "${spark_python_tests[*]}") - python/run-tests --testnames "$(IFS=,; echo "${spark_python_tests[*]}")" --python-executables python -popd diff --git a/ci/scripts/integration_turbodbc.sh b/ci/scripts/integration_turbodbc.sh deleted file mode 100755 index f56074358..000000000 --- a/ci/scripts/integration_turbodbc.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source_dir=${1} -build_dir=${2}/turbodbc - -# check that optional pyarrow modules are available -# because pytest would just skip the pyarrow tests -python -c "import pyarrow.orc" -python -c "import pyarrow.parquet" - -mkdir -p ${build_dir} -pushd ${build_dir} - -cmake -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ - -DCMAKE_CXX_FLAGS=${CXXFLAGS} \ - -DPYTHON_EXECUTABLE=$(which python) \ - -GNinja \ - ${source_dir} -ninja install - -# TODO(ARROW-5074) -export LD_LIBRARY_PATH="${ARROW_HOME}/lib:${LD_LIBRARY_PATH}" -export ODBCSYSINI="${source_dir}/travis/odbc/" - -service postgresql start -ctest --output-on-failure - -popd diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh deleted file mode 100755 index b8a7f7ced..000000000 --- a/ci/scripts/java_build.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/java -cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug} -with_docs=${3:-false} - -if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then - # Since some files for s390_64 are not available at maven central, - # download pre-build files from bintray and install them explicitly - mvn_install="mvn install:install-file" - wget="wget" - bintray_base_url="https://dl.bintray.com/apache/arrow" - - bintray_dir="protoc-binary" - group="com.google.protobuf" - artifact="protoc" - ver="3.7.1" - classifier="linux-s390_64" - extension="exe" - target=${artifact}-${ver}-${classifier}.${extension} - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} - # protoc requires libprotoc.so.18 libprotobuf.so.18 - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/libprotoc.so.18 - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/libprotobuf.so.18 - mkdir -p ${ARROW_HOME}/lib - cp lib*.so.18 ${ARROW_HOME}/lib - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ARROW_HOME}/lib - - bintray_dir="protoc-gen-grpc-java-binary" - group="io.grpc" - artifact="protoc-gen-grpc-java" - ver="1.30.2" - classifier="linux-s390_64" - extension="exe" - target=${artifact}-${ver}-${classifier}.${extension} - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} - - bintray_dir="netty-binary" - group="io.netty" - artifact="netty-transport-native-unix-common" - ver="4.1.48.Final" - classifier="linux-s390_64" - extension="jar" - target=${artifact}-${ver}-${classifier}.${extension} - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} - artifact="netty-transport-native-epoll" - extension="jar" - target=${artifact}-${ver}-${classifier}.${extension} - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} -fi - -mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -# Use `2 * ncores` threads -mvn="${mvn} -T 2C" - -pushd ${source_dir} - -${mvn} install - -if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then - ${mvn} -Pshade-flatbuffers install -fi - -if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then - ${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install -fi - -if [ "${ARROW_PLASMA}" = "ON" ]; then - pushd ${source_dir}/plasma - ${mvn} clean install - popd -fi - -if [ "${with_docs}" == "true" ]; then - ${mvn} -Dcheckstyle.skip=true install site -fi - -popd diff --git a/ci/scripts/java_test.sh b/ci/scripts/java_test.sh deleted file mode 100755 index da9e45280..000000000 --- a/ci/scripts/java_test.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/java -cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug} - -# For JNI and Plasma tests -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PLASMA_STORE=${ARROW_HOME}/bin/plasma-store-server - -mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -# Use `2 * ncores` threads -mvn="${mvn} -T 2C" - -pushd ${source_dir} - -${mvn} test - -if [ "${ARROW_JNI}" = "ON" ]; then - ${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${cpp_build_dir} -fi - -if [ "${ARROW_PLASMA}" = "ON" ]; then - pushd ${source_dir}/plasma - java -cp target/test-classes:target/classes \ - -Djava.library.path=${cpp_build_dir} \ - org.apache.arrow.plasma.PlasmaClientTest - popd -fi - -popd diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh deleted file mode 100755 index 10ceb41ee..000000000 --- a/ci/scripts/js_build.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/js -with_docs=${2:-false} - -pushd ${source_dir} - -yarn --frozen-lockfile -# TODO(kszucs): linting should be moved to archery -yarn lint:ci -yarn build - -if [ "${with_docs}" == "true" ]; then - yarn doc -fi - -popd diff --git a/ci/scripts/js_test.sh b/ci/scripts/js_test.sh deleted file mode 100755 index 345d6cb81..000000000 --- a/ci/scripts/js_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/js - -pushd ${source_dir} - -yarn lint -yarn test - -popd diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh deleted file mode 100755 index cb6ca30a6..000000000 --- a/ci/scripts/msys2_setup.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -target=$1 - -packages=() -case "${target}" in - cpp|c_glib|ruby) - packages+=(${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp) - packages+=(${MINGW_PACKAGE_PREFIX}-boost) - packages+=(${MINGW_PACKAGE_PREFIX}-brotli) - packages+=(${MINGW_PACKAGE_PREFIX}-ccache) - packages+=(${MINGW_PACKAGE_PREFIX}-clang) - packages+=(${MINGW_PACKAGE_PREFIX}-cmake) - packages+=(${MINGW_PACKAGE_PREFIX}-gcc) - packages+=(${MINGW_PACKAGE_PREFIX}-gflags) - packages+=(${MINGW_PACKAGE_PREFIX}-grpc) - packages+=(${MINGW_PACKAGE_PREFIX}-gtest) - packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc) - packages+=(${MINGW_PACKAGE_PREFIX}-llvm) - packages+=(${MINGW_PACKAGE_PREFIX}-lz4) - packages+=(${MINGW_PACKAGE_PREFIX}-ninja) - packages+=(${MINGW_PACKAGE_PREFIX}-polly) - packages+=(${MINGW_PACKAGE_PREFIX}-protobuf) - packages+=(${MINGW_PACKAGE_PREFIX}-python3-numpy) - packages+=(${MINGW_PACKAGE_PREFIX}-rapidjson) - packages+=(${MINGW_PACKAGE_PREFIX}-re2) - packages+=(${MINGW_PACKAGE_PREFIX}-snappy) - packages+=(${MINGW_PACKAGE_PREFIX}-thrift) - packages+=(${MINGW_PACKAGE_PREFIX}-zlib) - packages+=(${MINGW_PACKAGE_PREFIX}-zstd) - ;; -esac - -case "${target}" in - c_glib|ruby) - packages+=(${MINGW_PACKAGE_PREFIX}-gobject-introspection) - packages+=(${MINGW_PACKAGE_PREFIX}-gtk-doc) - packages+=(${MINGW_PACKAGE_PREFIX}-meson) - ;; -esac - -pacman \ - --needed \ - --noconfirm \ - --refresh \ - --sync \ - "${packages[@]}" - -"$(dirname $0)/ccache_setup.sh" -echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV diff --git a/ci/scripts/msys2_system_clean.sh b/ci/scripts/msys2_system_clean.sh deleted file mode 100755 index a356aee66..000000000 --- a/ci/scripts/msys2_system_clean.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -pacman \ - --cascade \ - --noconfirm \ - --nosave \ - --recursive \ - --remove \ - ${MINGW_PACKAGE_PREFIX}-clang-tools-extra \ - ${MINGW_PACKAGE_PREFIX}-gcc-ada \ - ${MINGW_PACKAGE_PREFIX}-gcc-fortran \ - ${MINGW_PACKAGE_PREFIX}-gcc-libgfortran \ - ${MINGW_PACKAGE_PREFIX}-gcc-objc \ - ${MINGW_PACKAGE_PREFIX}-libgccjit diff --git a/ci/scripts/msys2_system_upgrade.sh b/ci/scripts/msys2_system_upgrade.sh deleted file mode 100755 index aecd30893..000000000 --- a/ci/scripts/msys2_system_upgrade.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -# https://www.msys2.org/news/#2020-06-29-new-packagers -msys2_repo_base_url=https://repo.msys2.org/msys -# Mirror -msys2_repo_base_url=https://sourceforge.net/projects/msys2/files/REPOS/MSYS2 -msys2_keyring_pkg=msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz -for suffix in "" ".sig"; do - curl \ - --location \ - --remote-name \ - --show-error \ - --silent \ - ${msys2_repo_base_url}/x86_64/${msys2_keyring_pkg}${suffix} -done -pacman-key --verify ${msys2_keyring_pkg}.sig -pacman \ - --noconfirm \ - --upgrade \ - ${msys2_keyring_pkg} - - -pacman \ - --noconfirm \ - --refresh \ - --refresh \ - --sync \ - --sysupgrade \ - --sysupgrade diff --git a/ci/scripts/python_benchmark.sh b/ci/scripts/python_benchmark.sh deleted file mode 100755 index 3a35298dc..000000000 --- a/ci/scripts/python_benchmark.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Check the ASV benchmarking setup. -# Unfortunately this won't ensure that all benchmarks succeed -# (see https://github.com/airspeed-velocity/asv/issues/449) -source deactivate -conda create -y -q -n pyarrow_asv python=$PYTHON_VERSION -conda activate pyarrow_asv -pip install -q git+https://github.com/pitrou/asv.git@customize_commands - -export PYARROW_WITH_PARQUET=1 -export PYARROW_WITH_PLASMA=1 -export PYARROW_WITH_ORC=0 -export PYARROW_WITH_GANDIVA=0 - -pushd $ARROW_PYTHON_DIR -# Workaround for https://github.com/airspeed-velocity/asv/issues/631 -git fetch --depth=100 origin master:master -# Generate machine information (mandatory) -asv machine --yes -# Run benchmarks on the changeset being tested -asv run --no-pull --show-stderr --quick HEAD^! -popd # $ARROW_PYTHON_DIR diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh deleted file mode 100755 index ec6d723b2..000000000 --- a/ci/scripts/python_build.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/python -build_dir=${2}/python - -if [ ! -z "${CONDA_PREFIX}" ]; then - echo -e "===\n=== Conda environment for build\n===" - conda list -fi - -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja} -export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug} -export PYARROW_WITH_S3=${ARROW_S3:-OFF} -export PYARROW_WITH_ORC=${ARROW_ORC:-OFF} -export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF} -export PYARROW_WITH_HDFS=${ARROW_HDFS:-OFF} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF} -export PYARROW_WITH_PLASMA=${ARROW_PLASMA:-OFF} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF} -export PYARROW_WITH_DATASET=${ARROW_DATASET:-OFF} - -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} - -pushd ${source_dir} - -relative_build_dir=$(realpath --relative-to=. $build_dir) - -# not nice, but prevents mutating the mounted the source directory for docker -${PYTHON:-python} \ - setup.py build --build-base $build_dir \ - install --single-version-externally-managed \ - --record $relative_build_dir/record.txt - -popd diff --git a/ci/scripts/python_sdist_build.sh b/ci/scripts/python_sdist_build.sh deleted file mode 100755 index f9e9359b6..000000000 --- a/ci/scripts/python_sdist_build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -source_dir=${1}/python - -pushd ${source_dir} -export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-} -${PYTHON:-python} setup.py sdist -popd diff --git a/ci/scripts/python_sdist_test.sh b/ci/scripts/python_sdist_test.sh deleted file mode 100755 index 1388ca09e..000000000 --- a/ci/scripts/python_sdist_test.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -arrow_dir=${1} - -export ARROW_SOURCE_DIR=${arrow_dir} -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data - -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja} -export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug} -export PYARROW_WITH_S3=${ARROW_S3:-OFF} -export PYARROW_WITH_ORC=${ARROW_ORC:-OFF} -export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF} -export PYARROW_WITH_HDFS=${ARROW_HDFS:-OFF} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF} -export PYARROW_WITH_PLASMA=${ARROW_PLASMA:-OFF} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF} -export PYARROW_WITH_DATASET=${ARROW_DATASET:-OFF} - -# TODO: Users should not require ARROW_HOME and pkg-config to find Arrow C++. -# Related: ARROW-9171 -# unset ARROW_HOME -# apt purge -y pkg-config - -if [ -n "${PYARROW_VERSION:-}" ]; then - sdist="${arrow_dir}/python/dist/pyarrow-${PYARROW_VERSION}.tar.gz" -else - sdist=$(ls "${arrow_dir}/python/dist/pyarrow-*.tar.gz" | sort -r | head -n1) -fi -${PYTHON:-python} -m pip install ${sdist} - -pytest -r s ${PYTEST_ARGS:-} --pyargs pyarrow diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh deleted file mode 100755 index 80a9cdef4..000000000 --- a/ci/scripts/python_test.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} - -export ARROW_SOURCE_DIR=${arrow_dir} -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} - -# Enable some checks inside Python itself -export PYTHONDEVMODE=1 - -pytest -r s ${PYTEST_ARGS} --pyargs pyarrow diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh deleted file mode 100755 index 93e4939af..000000000 --- a/ci/scripts/python_wheel_macos_build.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1} -build_dir=${2} - -echo "=== (${PYTHON_VERSION}) Clear output directories and leftovers ===" -# Clear output directories and leftovers -rm -rf ${build_dir}/install -rm -rf ${source_dir}/python/dist -rm -rf ${source_dir}/python/build -rm -rf ${source_dir}/python/repaired_wheels -rm -rf ${source_dir}/python/pyarrow/*.so -rm -rf ${source_dir}/python/pyarrow/*.so.* - -echo "=== (${PYTHON_VERSION}) Set OSX SDK and C flags ===" -# Arrow is 64-bit-only at the moment -export CFLAGS="-fPIC -arch x86_64 ${CFLAGS//-arch i386/}" -export CXXFLAGS="-fPIC -arch x86_64 ${CXXFLAGS//-arch i386} -std=c++11" -export SDKROOT="$(xcrun --show-sdk-path)" - -echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" -: ${ARROW_DATASET:=ON} -: ${ARROW_FLIGHT:=ON} -: ${ARROW_GANDIVA:=OFF} -: ${ARROW_HDFS:=ON} -: ${ARROW_JEMALLOC:=ON} -: ${ARROW_MIMALLOC:=ON} -: ${ARROW_ORC:=ON} -: ${ARROW_PARQUET:=ON} -: ${ARROW_PLASMA:=ON} -: ${ARROW_S3:=ON} -: ${ARROW_TENSORFLOW:=ON} -: ${ARROW_WITH_BROTLI:=ON} -: ${ARROW_WITH_BZ2:=ON} -: ${ARROW_WITH_LZ4:=ON} -: ${ARROW_WITH_SNAPPY:=ON} -: ${ARROW_WITH_ZLIB:=ON} -: ${ARROW_WITH_ZSTD:=ON} -: ${CMAKE_BUILD_TYPE:=release} -: ${CMAKE_GENERATOR:=Ninja} -: ${VCPKG_FEATURE_FLAGS:=-manifests} -: ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}} - -mkdir -p ${build_dir}/build -pushd ${build_dir}/build -cmake \ - -DARROW_BUILD_SHARED=ON \ - -DARROW_BUILD_STATIC=OFF \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_DATASET=${ARROW_DATASET} \ - -DARROW_DEPENDENCY_SOURCE="VCPKG" \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_FLIGHT==${ARROW_FLIGHT} \ - -DARROW_GANDIVA=${ARROW_GANDIVA} \ - -DARROW_HDFS=${ARROW_HDFS} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC} \ - -DARROW_MIMALLOC=${ARROW_MIMALLOC} \ - -DARROW_ORC=${ARROW_ORC} \ - -DARROW_PACKAGE_KIND="python-wheel-macos" \ - -DARROW_PARQUET=${ARROW_PARQUET} \ - -DARROW_PLASMA=${ARROW_PLASMA} \ - -DARROW_PYTHON=ON \ - -DARROW_RPATH_ORIGIN=ON \ - -DARROW_S3=${ARROW_S3} \ - -DARROW_TENSORFLOW=${ARROW_TENSORFLOW} \ - -DARROW_USE_CCACHE=ON \ - -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI} \ - -DARROW_WITH_BZ2=${ARROW_WITH_BZ2} \ - -DARROW_WITH_LZ4=${ARROW_WITH_LZ4} \ - -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \ - -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \ - -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_INSTALL_PREFIX=${build_dir}/install \ - -DCMAKE_UNITY_BUILD=ON \ - -DOPENSSL_USE_STATIC_LIBS=ON \ - -DVCPKG_MANIFEST_MODE=OFF \ - -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \ - -G ${CMAKE_GENERATOR} \ - ${source_dir}/cpp -cmake --build . --target install -popd - -# Check that we don't expose any unwanted symbols -# check_arrow_visibility - -echo "=== (${PYTHON_VERSION}) Building wheel ===" -export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE} -export PYARROW_BUNDLE_ARROW_CPP=1 -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR} -export PYARROW_INSTALL_TESTS=1 -export PYARROW_WITH_DATASET=${ARROW_DATASET} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA} -export PYARROW_WITH_HDFS=${ARROW_HDFS} -export PYARROW_WITH_ORC=${ARROW_ORC} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET} -export PYARROW_WITH_PLASMA=${ARROW_PLASMA} -export PYARROW_WITH_S3=${ARROW_S3} -# PyArrow build configuration -export PKG_CONFIG_PATH=/usr/lib/pkgconfig:${build_dir}/install/lib/pkgconfig - -pushd ${source_dir}/python -python setup.py bdist_wheel -popd - -echo "=== (${PYTHON_VERSION}) Show dynamic libraries the wheel depend on ===" -deps=$(delocate-listdeps ${source_dir}/python/dist/*.whl) - -if echo $deps | grep -v "^@rpath/lib\(arrow\|gandiva\|parquet\|plasma\)"; then - echo "There are non-bundled shared library dependencies." - exit 1 -fi diff --git a/ci/scripts/python_wheel_macos_test.sh b/ci/scripts/python_wheel_macos_test.sh deleted file mode 100755 index 6ac8576d4..000000000 --- a/ci/scripts/python_wheel_macos_test.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1} - -: ${ARROW_S3:=ON} - -export PYARROW_TEST_CYTHON=OFF -export PYARROW_TEST_DATASET=ON -export PYARROW_TEST_GANDIVA=OFF -export PYARROW_TEST_HDFS=ON -export PYARROW_TEST_ORC=ON -export PYARROW_TEST_PANDAS=ON -export PYARROW_TEST_PARQUET=ON -export PYARROW_TEST_PLASMA=ON -export PYARROW_TEST_S3=${ARROW_S3} -export PYARROW_TEST_TENSORFLOW=ON -export PYARROW_TEST_FLIGHT=ON - -export ARROW_TEST_DATA=${source_dir}/testing/data -export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data - -# Install the built wheels -pip install ${source_dir}/python/dist/*.whl - -# Test that the modules are importable -python -c " -import pyarrow -import pyarrow._hdfs -import pyarrow.csv -import pyarrow.dataset -import pyarrow.flight -import pyarrow.fs -import pyarrow.json -import pyarrow.orc -import pyarrow.parquet -import pyarrow.plasma -" - -if [ "${PYARROW_TEST_S3}" == "ON" ]; then - python -c "import pyarrow._s3fs" -fi - -# Install testing dependencies -pip install -r ${source_dir}/python/requirements-wheel-test.txt - -# Execute unittest -pytest -r s --pyargs pyarrow diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh deleted file mode 100755 index 312e1c3b9..000000000 --- a/ci/scripts/python_wheel_manylinux_build.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -function check_arrow_visibility { - nm --demangle --dynamic /tmp/arrow-dist/lib/libarrow.so > nm_arrow.log - - # Filter out Arrow symbols and see if anything remains. - # '_init' and '_fini' symbols may or not be present, we don't care. - # (note we must ignore the grep exit status when no match is found) - grep ' T ' nm_arrow.log | grep -v -E '(arrow|\b_init\b|\b_fini\b)' | cat - > visible_symbols.log - - if [[ -f visible_symbols.log && `cat visible_symbols.log | wc -l` -eq 0 ]]; then - return 0 - else - echo "== Unexpected symbols exported by libarrow.so ==" - cat visible_symbols.log - echo "================================================" - - exit 1 - fi -} - -echo "=== (${PYTHON_VERSION}) Clear output directories and leftovers ===" -# Clear output directories and leftovers -rm -rf /tmp/arrow-build -rm -rf /arrow/python/dist -rm -rf /arrow/python/build -rm -rf /arrow/python/repaired_wheels -rm -rf /arrow/python/pyarrow/*.so -rm -rf /arrow/python/pyarrow/*.so.* - -echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" -: ${ARROW_DATASET:=ON} -: ${ARROW_FLIGHT:=ON} -: ${ARROW_GANDIVA:=OFF} -: ${ARROW_HDFS:=ON} -: ${ARROW_JEMALLOC:=ON} -: ${ARROW_MIMALLOC:=ON} -: ${ARROW_ORC:=ON} -: ${ARROW_PARQUET:=ON} -: ${ARROW_PLASMA:=ON} -: ${ARROW_S3:=ON} -: ${ARROW_TENSORFLOW:=ON} -: ${ARROW_WITH_BROTLI:=ON} -: ${ARROW_WITH_BZ2:=ON} -: ${ARROW_WITH_LZ4:=ON} -: ${ARROW_WITH_SNAPPY:=ON} -: ${ARROW_WITH_ZLIB:=ON} -: ${ARROW_WITH_ZSTD:=ON} -: ${CMAKE_BUILD_TYPE:=release} -: ${CMAKE_UNITY_BUILD:=ON} -: ${CMAKE_GENERATOR:=Ninja} -: ${VCPKG_FEATURE_FLAGS:=-manifests} -: ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}} - -mkdir /tmp/arrow-build -pushd /tmp/arrow-build -cmake \ - -DARROW_BROTLI_USE_SHARED=OFF \ - -DARROW_BUILD_SHARED=ON \ - -DARROW_BUILD_STATIC=OFF \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_DATASET=${ARROW_DATASET} \ - -DARROW_DEPENDENCY_SOURCE="VCPKG" \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_FLIGHT==${ARROW_FLIGHT} \ - -DARROW_GANDIVA=${ARROW_GANDIVA} \ - -DARROW_HDFS=${ARROW_HDFS} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC} \ - -DARROW_MIMALLOC=${ARROW_MIMALLOC} \ - -DARROW_ORC=${ARROW_ORC} \ - -DARROW_PACKAGE_KIND="python-wheel-manylinux${MANYLINUX_VERSION}" \ - -DARROW_PARQUET=${ARROW_PARQUET} \ - -DARROW_PLASMA=${ARROW_PLASMA} \ - -DARROW_PYTHON=ON \ - -DARROW_RPATH_ORIGIN=ON \ - -DARROW_S3=${ARROW_S3} \ - -DARROW_TENSORFLOW=${ARROW_TENSORFLOW} \ - -DARROW_USE_CCACHE=ON \ - -DARROW_UTF8PROC_USE_SHARED=OFF \ - -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI} \ - -DARROW_WITH_BZ2=${ARROW_WITH_BZ2} \ - -DARROW_WITH_LZ4=${ARROW_WITH_LZ4} \ - -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \ - -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \ - -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ - -DOPENSSL_USE_STATIC_LIBS=ON \ - -DVCPKG_MANIFEST_MODE=OFF \ - -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \ - -G ${CMAKE_GENERATOR} \ - /arrow/cpp -cmake --build . --target install -popd - -# Check that we don't expose any unwanted symbols -check_arrow_visibility - -echo "=== (${PYTHON_VERSION}) Building wheel ===" -export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE} -export PYARROW_BUNDLE_ARROW_CPP=1 -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR} -export PYARROW_INSTALL_TESTS=1 -export PYARROW_WITH_DATASET=${ARROW_DATASET} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA} -export PYARROW_WITH_HDFS=${ARROW_HDFS} -export PYARROW_WITH_ORC=${ARROW_ORC} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET} -export PYARROW_WITH_PLASMA=${ARROW_PLASMA} -export PYARROW_WITH_S3=${ARROW_S3} -# PyArrow build configuration -export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/tmp/arrow-dist/lib/pkgconfig - -pushd /arrow/python -python setup.py bdist_wheel - -echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux${MANYLINUX_VERSION} ===" -auditwheel repair -L . dist/pyarrow-*.whl -w repaired_wheels -popd diff --git a/ci/scripts/python_wheel_manylinux_test.sh b/ci/scripts/python_wheel_manylinux_test.sh deleted file mode 100755 index 21987748f..000000000 --- a/ci/scripts/python_wheel_manylinux_test.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -x -set -o pipefail - -case $# in - 1) KIND="$1" - case $KIND in - imports|unittests) ;; - *) echo "Invalid argument: '${KIND}', valid options are 'imports', 'unittests'" - exit 1 - ;; - esac - ;; - *) echo "Usage: $0 imports|unittests" - exit 1 - ;; -esac - -export PYARROW_TEST_CYTHON=OFF -export PYARROW_TEST_DATASET=ON -export PYARROW_TEST_GANDIVA=OFF -export PYARROW_TEST_HDFS=ON -export PYARROW_TEST_ORC=ON -export PYARROW_TEST_PANDAS=ON -export PYARROW_TEST_PARQUET=ON -export PYARROW_TEST_PLASMA=ON -export PYARROW_TEST_S3=ON -export PYARROW_TEST_TENSORFLOW=ON -export PYARROW_TEST_FLIGHT=ON - -export ARROW_TEST_DATA=/arrow/testing/data -export PARQUET_TEST_DATA=/arrow/submodules/parquet-testing/data - -# Install the built wheels -pip install /arrow/python/repaired_wheels/*.whl - -if [ "${KIND}" == "imports" ]; then - # Test that the modules are importable - python -c " -import pyarrow -import pyarrow._hdfs -import pyarrow._s3fs -import pyarrow.csv -import pyarrow.dataset -import pyarrow.flight -import pyarrow.fs -import pyarrow.json -import pyarrow.orc -import pyarrow.parquet -import pyarrow.plasma" -elif [ "${KIND}" == "unittests" ]; then - # Execute unittest, test dependencies must be installed - pytest -r s --pyargs pyarrow -fi diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat deleted file mode 100644 index 23be7f512..000000000 --- a/ci/scripts/python_wheel_windows_build.bat +++ /dev/null @@ -1,109 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@echo on - -echo "Building windows wheel..." - -call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" - -echo "=== (%PYTHON_VERSION%) Clear output directories and leftovers ===" -del /s /q C:\arrow-build -del /s /q C:\arrow-dist -del /s /q C:\arrow\python\dist -del /s /q C:\arrow\python\build -del /s /q C:\arrow\python\pyarrow\*.so -del /s /q C:\arrow\python\pyarrow\*.so.* - -echo "=== (%PYTHON_VERSION%) Building Arrow C++ libraries ===" -set ARROW_DATASET=ON -set ARROW_FLIGHT=ON -set ARROW_GANDIVA=OFF -set ARROW_HDFS=ON -set ARROW_ORC=OFF -set ARROW_PARQUET=ON -set ARROW_MIMALLOC=ON -set ARROW_S3=ON -set ARROW_TENSORFLOW=ON -set ARROW_WITH_BROTLI=ON -set ARROW_WITH_BZ2=ON -set ARROW_WITH_LZ4=ON -set ARROW_WITH_SNAPPY=ON -set ARROW_WITH_ZLIB=ON -set ARROW_WITH_ZSTD=ON -set CMAKE_UNITY_BUILD=ON -set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 -set VCPKG_FEATURE_FLAGS=-manifests - -mkdir C:\arrow-build -pushd C:\arrow-build -cmake ^ - -DARROW_BUILD_SHARED=ON ^ - -DARROW_BUILD_STATIC=OFF ^ - -DARROW_BUILD_TESTS=OFF ^ - -DARROW_CXXFLAGS="/MP" ^ - -DARROW_DATASET=%ARROW_DATASET% ^ - -DARROW_DEPENDENCY_SOURCE=VCPKG ^ - -DARROW_DEPENDENCY_USE_SHARED=OFF ^ - -DARROW_FLIGHT=%ARROW_FLIGHT% ^ - -DARROW_GANDIVA=%ARROW_GANDIVA% ^ - -DARROW_HDFS=%ARROW_HDFS% ^ - -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ - -DARROW_ORC=%ARROW_ORC% ^ - -DARROW_PACKAGE_KIND="python-wheel-windows" ^ - -DARROW_PARQUET=%ARROW_PARQUET% ^ - -DARROW_PYTHON=ON ^ - -DARROW_S3=%ARROW_S3% ^ - -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ - -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^ - -DARROW_WITH_BZ2=%ARROW_WITH_BZ2% ^ - -DARROW_WITH_LZ4=%ARROW_WITH_LZ4% ^ - -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^ - -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^ - -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^ - -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ - -DCMAKE_CXX_COMPILER=clcache ^ - -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ - -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^ - -DMSVC_LINK_VERBOSE=ON ^ - -DVCPKG_MANIFEST_MODE=OFF ^ - -DVCPKG_TARGET_TRIPLET=x64-windows-static-md-%CMAKE_BUILD_TYPE% ^ - -G "%CMAKE_GENERATOR%" ^ - C:\arrow\cpp || exit /B -cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B -popd - -echo "=== (%PYTHON_VERSION%) Building wheel ===" -set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE% -set PYARROW_BUNDLE_ARROW_CPP=ON -set PYARROW_BUNDLE_BOOST=OFF -set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR% -set PYARROW_INSTALL_TESTS=ON -set PYARROW_WITH_DATASET=%ARROW_DATASET% -set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT% -set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA% -set PYARROW_WITH_HDFS=%ARROW_HDFS% -set PYARROW_WITH_ORC=%ARROW_ORC% -set PYARROW_WITH_PARQUET=%ARROW_PARQUET% -set PYARROW_WITH_S3=%ARROW_S3% -set ARROW_HOME=C:\arrow-dist - -pushd C:\arrow\python -@REM bundle the msvc runtime -cp "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Redist\MSVC\14.16.27012\x64\Microsoft.VC141.CRT\msvcp140.dll" pyarrow\ -python setup.py bdist_wheel || exit /B -popd diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat deleted file mode 100755 index 8352e5862..000000000 --- a/ci/scripts/python_wheel_windows_test.bat +++ /dev/null @@ -1,54 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@echo on - -set PYARROW_TEST_CYTHON=OFF -set PYARROW_TEST_DATASET=ON -set PYARROW_TEST_GANDIVA=OFF -set PYARROW_TEST_HDFS=ON -set PYARROW_TEST_ORC=OFF -set PYARROW_TEST_PANDAS=ON -set PYARROW_TEST_PARQUET=ON -set PYARROW_TEST_PLASMA=OFF -set PYARROW_TEST_S3=OFF -set PYARROW_TEST_TENSORFLOW=ON -set PYARROW_TEST_FLIGHT=ON - -set ARROW_TEST_DATA=C:\arrow\testing\data -set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data - -@REM Install the built wheels -python -m pip install numpy -python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B - -@REM Test that the modules are importable -python -c "import pyarrow" -python -c "import pyarrow._hdfs" -python -c "import pyarrow._s3fs" -python -c "import pyarrow.csv" -python -c "import pyarrow.dataset" -python -c "import pyarrow.flight" -python -c "import pyarrow.fs" -python -c "import pyarrow.json" -python -c "import pyarrow.parquet" - -@REM Install testing dependencies -pip install -r C:\arrow\python\requirements-wheel-test.txt - -@REM Execute unittest -pytest -r s --pyargs pyarrow diff --git a/ci/scripts/r_build.sh b/ci/scripts/r_build.sh deleted file mode 100755 index d1907a87c..000000000 --- a/ci/scripts/r_build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=R} -source_dir=${1}/r -with_docs=${2:-false} - -pushd ${source_dir} - -${R_BIN} CMD INSTALL . - -if [ "${with_docs}" == "true" ]; then - ${R_BIN} -e "pkgdown::build_site(install = FALSE)" -fi - -popd \ No newline at end of file diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh deleted file mode 100755 index 7e9d2eac7..000000000 --- a/ci/scripts/r_deps.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=R} - -source_dir=${1}/r - -pushd ${source_dir} - -# Install R package dependencies -${R_BIN} -e "install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck', 'sys'))" -${R_BIN} -e "remotes::install_deps(dependencies = TRUE)" - -popd diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh deleted file mode 100755 index 3e553fe9e..000000000 --- a/ci/scripts/r_docker_configure.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=R} - -# The Dockerfile should have put this file here -if [ -f "/arrow/ci/etc/rprofile" ]; then - # Ensure parallel R package installation, set CRAN repo mirror, - # and use pre-built binaries where possible - cat /arrow/ci/etc/rprofile >> $(${R_BIN} RHOME)/etc/Rprofile.site -fi - -# Ensure parallel compilation of C/C++ code -echo "MAKEFLAGS=-j$(${R_BIN} -s -e 'cat(parallel::detectCores())')" >> $(${R_BIN} RHOME)/etc/Makeconf - -# Special hacking to try to reproduce quirks on fedora-clang-devel on CRAN -# which uses a bespoke clang compiled to use libc++ -# https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang -if [ "$RHUB_PLATFORM" = "linux-x86_64-fedora-clang" ]; then - dnf install -y libcxx-devel - sed -i.bak -E -e 's/(CXX1?1? =.*)/\1 -stdlib=libc++/g' $(${R_BIN} RHOME)/etc/Makeconf - rm -rf $(${R_BIN} RHOME)/etc/Makeconf.bak -fi - -# Special hacking to try to reproduce quirks on centos using non-default build -# tooling. -if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then - if [ "`which dnf`" ]; then - dnf install -y centos-release-scl - dnf install -y "devtoolset-$DEVTOOLSET_VERSION" - else - yum install -y centos-release-scl - yum install -y "devtoolset-$DEVTOOLSET_VERSION" - fi -fi - -# Install openssl for S3 support -if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then - if [ "`which dnf`" ]; then - dnf install -y libcurl-devel openssl-devel - elif [ "`which yum`" ]; then - yum install -y libcurl-devel openssl-devel - elif [ "`which zypper`" ]; then - zypper install -y libcurl-devel libopenssl-devel - else - apt-get update - apt-get install -y libcurl4-openssl-dev libssl-dev - fi - - # The Dockerfile should have put this file here - if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then - /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local - fi -fi - -# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786 -Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}' - -if [ "`which curl`" ]; then - # We need this on R >= 4.0 - curl -L https://sourceforge.net/projects/checkbaskisms/files/2.0.0.2/checkbashisms/download > /usr/local/bin/checkbashisms - chmod 755 /usr/local/bin/checkbashisms -fi diff --git a/ci/scripts/r_pkgdown_check.sh b/ci/scripts/r_pkgdown_check.sh deleted file mode 100755 index 327480a6b..000000000 --- a/ci/scripts/r_pkgdown_check.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Use this script to prevent errors in the pkgdown site being rendered due to missing YAML entries - -# all .Rd files in the repo -all_rd_files=`find ./r/man -maxdepth 1 -name "*.Rd" | sed -e 's/.\/r\/man\///g' | sed -e 's/.Rd//g' | sort` - -# .Rd files to exclude from search (i.e. are internal) -exclusions=`grep "\keyword{internal}" -rl ./r/man --include=*.Rd | sed -e 's/.\/r\/man\///g' | sed -e 's/.Rd//g' | sort` - -# .Rd files to check against pkgdown.yml -rd_files=`echo ${exclusions[@]} ${all_rd_files[@]} | tr ' ' '\n' | sort | uniq -u` - -# pkgdown sections -pkgdown_sections=`awk '/^[^ ]/{ f=/reference:/; next } f{ if (sub(/:$/,"")) pkg=$2; else print pkg, $2 }' ./r/_pkgdown.yml | grep -v "title:" | sort` - -# get things that appear in man files that don't appear in pkgdown sections -pkgdown_missing=`echo ${pkgdown_sections[@]} ${pkgdown_sections[@]} ${rd_files[@]} | tr ' ' '\n' | sort | uniq -u` - -# if any sections are missing raise an error -if ([ ${#pkgdown_missing} -ge 1 ]); then - echo "Error! $pkgdown_missing missing from ./r/_pkgdown.yml" - exit 1 -fi diff --git a/ci/scripts/r_sanitize.sh b/ci/scripts/r_sanitize.sh deleted file mode 100755 index 89963eb2d..000000000 --- a/ci/scripts/r_sanitize.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=RDsan} - -source_dir=${1}/r - -${R_BIN} CMD INSTALL ${source_dir} -pushd ${source_dir}/tests - -export TEST_R_WITH_ARROW=TRUE -export UBSAN_OPTIONS="print_stacktrace=1,suppressions=/arrow/r/tools/ubsan.supp" -${R_BIN} < testthat.R > testthat.out 2>&1 || { cat testthat.out; exit 1; } - -cat testthat.out -if grep -q "runtime error" testthat.out; then - exit 1 -fi -popd diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh deleted file mode 100755 index 1f6a8721e..000000000 --- a/ci/scripts/r_test.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=R} - -source_dir=${1}/r - -pushd ${source_dir} - -printenv - -if [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then - export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} - export R_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} -fi -export _R_CHECK_COMPILATION_FLAGS_KNOWN_=${ARROW_R_CXXFLAGS} -if [ "$ARROW_R_DEV" = "TRUE" ]; then - # These are used in the Arrow C++ build and are not a problem - export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_} -Wno-attributes -msse4.2" - # Note that NOT_CRAN=true means (among other things) that optional dependencies are built - export NOT_CRAN=true -fi -: ${TEST_R_WITH_ARROW:=TRUE} -export TEST_R_WITH_ARROW=$TEST_R_WITH_ARROW - -export _R_CHECK_CRAN_INCOMING_REMOTE_=FALSE -# --run-donttest was used in R < 4.0, this is used now -export _R_CHECK_DONTTEST_EXAMPLES_=$TEST_R_WITH_ARROW -# Not all Suggested packages are needed for checking, so in case they aren't installed don't fail -export _R_CHECK_FORCE_SUGGESTS_=FALSE -export _R_CHECK_LIMIT_CORES_=FALSE -export _R_CHECK_TESTS_NLINES_=0 - -# By default, aws-sdk tries to contact a non-existing local ip host -# to retrieve metadata. Disable this so that S3FileSystem tests run faster. -export AWS_EC2_METADATA_DISABLED=TRUE - -# Hack so that texlive2020 doesn't pollute the home dir -export TEXMFCONFIG=/tmp/texmf-config -export TEXMFVAR=/tmp/texmf-var - -if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then - # enable the devtoolset version to use it - source /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable -fi - -# Make sure we aren't writing to the home dir (CRAN _hates_ this but there is no official check) -BEFORE=$(ls -alh ~/) - -SCRIPT="as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true') - run_donttest <- identical(tolower(Sys.getenv('_R_CHECK_DONTTEST_EXAMPLES_', 'true')), 'true') - if (as_cran) { - rcmdcheck::rcmdcheck(args = c('--as-cran', if (run_donttest) '--run-donttest'), error_on = 'warning', check_dir = 'check', timeout = 3600) - } else { - if (nzchar(Sys.which('minio'))) { - message('Running minio for S3 tests (if build supports them)') - minio_dir <- tempfile() - dir.create(minio_dir) - pid <- sys::exec_background('minio', c('server', minio_dir)) - on.exit(tools::pskill(pid)) - } - rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', if (run_donttest) '--run-donttest'), error_on = 'warning', check_dir = 'check', timeout = 3600) - }" -echo "$SCRIPT" | ${R_BIN} --no-save - -AFTER=$(ls -alh ~/) -if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then - ls -alh ~/.cmake/packages - exit 1 -fi -popd diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh deleted file mode 100755 index 9988dfb64..000000000 --- a/ci/scripts/r_windows_build.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${ARROW_HOME:=$(pwd)} -# Make sure it is absolute and exported -export ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" - -if [ "$RTOOLS_VERSION" = "35" ]; then - # Use rtools-backports if building with rtools35 - curl https://raw.githubusercontent.com/r-windows/rtools-backports/master/pacman.conf > /etc/pacman.conf - # Update keys: https://www.msys2.org/news/#2020-06-29-new-packagers - msys2_repo_base_url=https://repo.msys2.org/msys - curl -OSsL "${msys2_repo_base_url}/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" - pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz - pacman --noconfirm -Scc - pacman --noconfirm -Syy - # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) - RWINLIB_LIB_DIR="lib-4.9.3" -else - # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN - # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf - # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" - # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz - # pacman --noconfirm -Scc - - pacman --noconfirm -Syy - RWINLIB_LIB_DIR="lib" -fi - -cp $ARROW_HOME/ci/scripts/PKGBUILD . -printenv -makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild - -VERSION=$(grep Version $ARROW_HOME/r/DESCRIPTION | cut -d " " -f 2) -DST_DIR="arrow-$VERSION" - -# Collect the build artifacts and make the shape of zip file that rwinlib expects -ls -mkdir -p build -mv mingw* build -cd build - -# This may vary by system/CI provider -MSYS_LIB_DIR="/c/rtools40" - -ls $MSYS_LIB_DIR/mingw64/lib/ -ls $MSYS_LIB_DIR/mingw32/lib/ - -# Untar the two builds we made -ls *.xz | xargs -n 1 tar -xJf -mkdir -p $DST_DIR -# Grab the headers from one, either one is fine -# (if we're building twice to combine old and new toolchains, this may already exist) -if [ ! -d $DST_DIR/include ]; then - mv mingw64/include $DST_DIR -fi - -# Make the rest of the directory structure -# lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) -mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/x64 -mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/i386 -# lib is for the new gcc 8 toolchain (Rtools 4.0) -mkdir -p $DST_DIR/lib/x64 -mkdir -p $DST_DIR/lib/i386 - -# Move the 64-bit versions of libarrow into the expected location -mv mingw64/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/x64 -# Same for the 32-bit versions -mv mingw32/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 - -# These may be from https://dl.bintray.com/rtools/backports/ -cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x64 -cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 - -# These are from https://dl.bintray.com/rtools/mingw{32,64}/ -cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/x64 -cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/i386 - -# Create build artifact -zip -r ${DST_DIR}.zip $DST_DIR - -# Copy that to a file name/path that does not vary by version number so we -# can easily find it in the R package tests on CI -cp ${DST_DIR}.zip ../libarrow.zip diff --git a/ci/scripts/release_test.sh b/ci/scripts/release_test.sh deleted file mode 100755 index ae2ab3288..000000000 --- a/ci/scripts/release_test.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -arrow_dir=${1} - -pushd ${arrow_dir} - -dev/release/run-test.rb - -popd diff --git a/ci/scripts/ruby_test.sh b/ci/scripts/ruby_test.sh deleted file mode 100755 index 03d20e198..000000000 --- a/ci/scripts/ruby_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/ruby -build_dir=${2}/ruby - -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig -export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 - -rake -f ${source_dir}/Rakefile BUILD_DIR=${build_dir} USE_BUNDLER=yes diff --git a/ci/scripts/util_checkout.sh b/ci/scripts/util_checkout.sh deleted file mode 100755 index 25fe69aa1..000000000 --- a/ci/scripts/util_checkout.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# this script is github actions specific to check out the submodules and tags - -# TODO(kszucs): remove it once the "submodules: recursive" feature is released -auth_header="$(git config --local --get http.https://github.com/.extraheader)" -git submodule sync --recursive -git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 - -# fetch all the tags -git fetch --depth=1 origin +refs/tags/*:refs/tags/* diff --git a/ci/scripts/util_cleanup.sh b/ci/scripts/util_cleanup.sh deleted file mode 100755 index 3a13a1a78..000000000 --- a/ci/scripts/util_cleanup.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This script is Github Actions-specific to free up disk space, -# to avoid disk full errors on some builds - -if [ $RUNNER_OS = "Linux" ]; then - df -h - - # remove swap - sudo swapoff -a - sudo rm -f /swapfile - - # clean apt cache - sudo apt clean - - # remove haskell, consumes 8.6 GB - sudo rm -rf /opt/ghc - - # 1 GB - sudo rm -rf /home/linuxbrew/.linuxbrew - - # 1+ GB - sudo rm -rf /opt/hostedtoolcache/CodeQL - - # 1+ GB - sudo rm -rf /usr/share/swift - - # 12 GB, but takes a lot of time to delete - #sudo rm -rf /usr/local/lib/android - - # remove cached docker images, around 13 GB - docker rmi $(docker image ls -aq) - - # NOTE: /usr/share/dotnet is 25 GB -fi - -df -h diff --git a/ci/scripts/util_download_apache.sh b/ci/scripts/util_download_apache.sh deleted file mode 100755 index 5d37f9c03..000000000 --- a/ci/scripts/util_download_apache.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -x -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -tarball_path=$1 -target_dir=$2 - -APACHE_MIRRORS=( - "http://www.apache.org/dyn/closer.cgi?action=download&filename=" - "https://downloads.apache.org" - "https://apache.claz.org" - "https://apache.cs.utah.edu" - "https://apache.mirrors.lucidnetworks.net" - "https://apache.osuosl.org" - "https://ftp.wayne.edu/apache" - "https://mirror.olnevhost.net/pub/apache" - "https://mirrors.gigenet.com/apache" - "https://mirrors.koehn.com/apache" - "https://mirrors.ocf.berkeley.edu/apache" - "https://mirrors.sonic.net/apache" - "https://us.mirrors.quenda.co/apache" -) - -mkdir -p "${target_dir}" - -for mirror in ${APACHE_MIRRORS[*]} -do - curl -SL "${mirror}/${tarball_path}" | tar -xzf - -C "${target_dir}" - if [ $? == 0 ]; then - exit 0 - fi -done - -exit 1 diff --git a/ci/scripts/util_wait_for_it.sh b/ci/scripts/util_wait_for_it.sh deleted file mode 100755 index 51ce816eb..000000000 --- a/ci/scripts/util_wait_for_it.sh +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env bash - -# The MIT License (MIT) -# Copyright (c) 2016 Giles Hall -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Use this script to test if a given TCP host/port are available - -cmdname=$(basename $0) - -echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $TIMEOUT -gt 0 ]]; then - echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" - else - echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" - fi - start_ts=$(date +%s) - while : - do - if [[ $ISBUSY -eq 1 ]]; then - nc -z $HOST $PORT - result=$? - else - (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 - result=$? - fi - if [[ $result -eq 0 ]]; then - end_ts=$(date +%s) - echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" - break - fi - sleep 1 - done - return $result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $QUIET -eq 1 ]]; then - timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - else - timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - fi - PID=$! - trap "kill -INT -$PID" INT - wait $PID - RESULT=$? - if [[ $RESULT -ne 0 ]]; then - echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" - fi - return $RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - hostport=(${1//:/ }) - HOST=${hostport[0]} - PORT=${hostport[1]} - shift 1 - ;; - --child) - CHILD=1 - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -s | --strict) - STRICT=1 - shift 1 - ;; - -h) - HOST="$2" - if [[ $HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - HOST="${1#*=}" - shift 1 - ;; - -p) - PORT="$2" - if [[ $PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - PORT="${1#*=}" - shift 1 - ;; - -t) - TIMEOUT="$2" - if [[ $TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$HOST" == "" || "$PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -TIMEOUT=${TIMEOUT:-15} -STRICT=${STRICT:-0} -CHILD=${CHILD:-0} -QUIET=${QUIET:-0} - -# check to see if timeout is from busybox? -# check to see if timeout is from busybox? -TIMEOUT_PATH=$(realpath $(which timeout)) -if [[ $TIMEOUT_PATH =~ "busybox" ]]; then - ISBUSY=1 - BUSYTIMEFLAG="-t" -else - ISBUSY=0 - BUSYTIMEFLAG="" -fi - -if [[ $CHILD -gt 0 ]]; then - wait_for - RESULT=$? - exit $RESULT -else - if [[ $TIMEOUT -gt 0 ]]; then - wait_for_wrapper - RESULT=$? - else - wait_for - RESULT=$? - fi -fi - -if [[ $CLI != "" ]]; then - if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then - echoerr "$cmdname: strict mode, refusing to execute subprocess" - exit $RESULT - fi - exec "${CLI[@]}" -else - exit $RESULT -fi diff --git a/ci/vcpkg/arm64-linux-static-debug.cmake b/ci/vcpkg/arm64-linux-static-debug.cmake deleted file mode 100644 index 6fea43694..000000000 --- a/ci/vcpkg/arm64-linux-static-debug.cmake +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(VCPKG_TARGET_ARCHITECTURE arm64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) -set(VCPKG_CMAKE_SYSTEM_NAME Linux) -set(VCPKG_BUILD_TYPE debug) - -if(NOT CMAKE_HOST_SYSTEM_PROCESSOR) - execute_process(COMMAND "uname" "-m" - OUTPUT_VARIABLE CMAKE_HOST_SYSTEM_PROCESSOR - OUTPUT_STRIP_TRAILING_WHITESPACE) -endif() diff --git a/ci/vcpkg/arm64-linux-static-release.cmake b/ci/vcpkg/arm64-linux-static-release.cmake deleted file mode 100644 index 4012848b8..000000000 --- a/ci/vcpkg/arm64-linux-static-release.cmake +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(VCPKG_TARGET_ARCHITECTURE arm64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) -set(VCPKG_CMAKE_SYSTEM_NAME Linux) -set(VCPKG_BUILD_TYPE release) - -if(NOT CMAKE_HOST_SYSTEM_PROCESSOR) - execute_process(COMMAND "uname" "-m" - OUTPUT_VARIABLE CMAKE_HOST_SYSTEM_PROCESSOR - OUTPUT_STRIP_TRAILING_WHITESPACE) -endif() diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch deleted file mode 100644 index 14b967869..000000000 --- a/ci/vcpkg/ports.patch +++ /dev/null @@ -1,63 +0,0 @@ -diff --git a/ports/aws-c-common/portfile.cmake b/ports/aws-c-common/portfile.cmake -index f3704ef05..3af543058 100644 ---- a/ports/aws-c-common/portfile.cmake -+++ b/ports/aws-c-common/portfile.cmake -@@ -1,8 +1,8 @@ - vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-common -- REF 4a21a1c0757083a16497fea27886f5f20ccdf334 # v0.4.56 -- SHA512 68898a8ac15d5490f45676eabfbe0df9e45370a74c543a28909fd0d85fed48dfcf4bcd6ea2d01d1a036dd352e2e4e0b08c48c63ab2a2b477fe150b46a827136e -+ REF 13adef72b7813ec878817c6d50a7a3f241015d8a # v0.4.57 -+ SHA512 28256522ac6af544d7464e3e7dcd4dc802ae2b09728bf8f167f86a6487bb756d0cad5eb4a2480610b2967b9c24c4a7f70621894517aa2828ffdeb0479453803b - HEAD_REF master - PATCHES - disable-error-4068.patch # This patch fixes dependency port compilation failure -diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake -index 6e18aecd0..2ccecf33c 100644 ---- a/ports/curl/portfile.cmake -+++ b/ports/curl/portfile.cmake -@@ -76,6 +76,8 @@ vcpkg_configure_cmake( - -DCMAKE_DISABLE_FIND_PACKAGE_Perl=ON - -DENABLE_DEBUG=ON - -DCURL_CA_FALLBACK=ON -+ -DCURL_CA_PATH=none -+ -DCURL_CA_BUNDLE=none - ) - - vcpkg_install_cmake() -diff --git a/ports/snappy/portfile.cmake b/ports/snappy/portfile.cmake -index 75dd13302..84345c7ca 100644 ---- a/ports/snappy/portfile.cmake -+++ b/ports/snappy/portfile.cmake -@@ -4,6 +4,7 @@ vcpkg_from_github( - REF 537f4ad6240e586970fe554614542e9717df7902 # 1.1.8 - SHA512 555d3b69a6759592736cbaae8f41654f0cf14e8be693b5dde37640191e53daec189f895872557b173e905d10681ef502f3e6ed8566811add963ffef96ce4016d - HEAD_REF master -+ PATCHES "snappy-disable-bmi.patch" - ) - - vcpkg_configure_cmake( -diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch -new file mode 100644 -index 000000000..2cbb1533a ---- /dev/null -+++ b/ports/snappy/snappy-disable-bmi.patch -@@ -0,0 +1,17 @@ -+--- snappy.cc 2020-06-27 17:38:49.718993748 -0500 -++++ snappy.cc 2020-06-27 17:37:57.543268213 -0500 -+@@ -717,14 +717,10 @@ -+ static inline uint32 ExtractLowBytes(uint32 v, int n) { -+ assert(n >= 0); -+ assert(n <= 4); -+-#if SNAPPY_HAVE_BMI2 -+- return _bzhi_u32(v, 8 * n); -+-#else -+ // This needs to be wider than uint32 otherwise `mask << 32` will be -+ // undefined. -+ uint64 mask = 0xffffffff; -+ return v & ~(mask << (8 * n)); -+-#endif -+ } -+ -+ static inline bool LeftShiftOverflows(uint8 value, uint32 shift) { diff --git a/ci/vcpkg/x64-linux-static-debug.cmake b/ci/vcpkg/x64-linux-static-debug.cmake deleted file mode 100644 index 3acee2ee4..000000000 --- a/ci/vcpkg/x64-linux-static-debug.cmake +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(VCPKG_TARGET_ARCHITECTURE x64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) - -set(VCPKG_CMAKE_SYSTEM_NAME Linux) - -set(VCPKG_BUILD_TYPE debug) diff --git a/ci/vcpkg/x64-linux-static-release.cmake b/ci/vcpkg/x64-linux-static-release.cmake deleted file mode 100644 index c2caa49fa..000000000 --- a/ci/vcpkg/x64-linux-static-release.cmake +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(VCPKG_TARGET_ARCHITECTURE x64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) - -set(VCPKG_CMAKE_SYSTEM_NAME Linux) - -set(VCPKG_BUILD_TYPE release) diff --git a/ci/vcpkg/x64-osx-static-debug.cmake b/ci/vcpkg/x64-osx-static-debug.cmake deleted file mode 100644 index e8a321ec7..000000000 --- a/ci/vcpkg/x64-osx-static-debug.cmake +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(VCPKG_TARGET_ARCHITECTURE x64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) - -set(VCPKG_CMAKE_SYSTEM_NAME Darwin) -set(VCPKG_OSX_ARCHITECTURES x86_64) - -set(VCPKG_BUILD_TYPE debug) diff --git a/ci/vcpkg/x64-osx-static-release.cmake b/ci/vcpkg/x64-osx-static-release.cmake deleted file mode 100644 index 956d5b92e..000000000 --- a/ci/vcpkg/x64-osx-static-release.cmake +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(VCPKG_TARGET_ARCHITECTURE x64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) - -set(VCPKG_CMAKE_SYSTEM_NAME Darwin) -set(VCPKG_OSX_ARCHITECTURES x86_64) - -set(VCPKG_BUILD_TYPE release) diff --git a/ci/vcpkg/x64-windows-static-md-debug.cmake b/ci/vcpkg/x64-windows-static-md-debug.cmake deleted file mode 100644 index 3eae3cfda..000000000 --- a/ci/vcpkg/x64-windows-static-md-debug.cmake +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(VCPKG_TARGET_ARCHITECTURE x64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) - -set(VCPKG_BUILD_TYPE debug) diff --git a/ci/vcpkg/x64-windows-static-md-release.cmake b/ci/vcpkg/x64-windows-static-md-release.cmake deleted file mode 100644 index b8dfbc884..000000000 --- a/ci/vcpkg/x64-windows-static-md-release.cmake +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(VCPKG_TARGET_ARCHITECTURE x64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) - -set(VCPKG_BUILD_TYPE release) From 36ba83ec269cee28dd3701605249159af8718ebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Sat, 12 Oct 2024 00:30:14 +0100 Subject: [PATCH 4/5] Code cleanup, move examples, remove unused files (#1075) Relates to: #1066 & #1067 --- .readthedocs.yml | 19 -- ballista/client/src/columnar_batch.rs | 165 ------------------ ballista/client/src/lib.rs | 1 - .../remote-dataframe.rs} | 0 .../bin/sql.rs => examples/remote-sql.rs} | 0 5 files changed, 185 deletions(-) delete mode 100644 .readthedocs.yml delete mode 100644 ballista/client/src/columnar_batch.rs rename examples/{src/bin/dataframe.rs => examples/remote-dataframe.rs} (100%) rename examples/{src/bin/sql.rs => examples/remote-sql.rs} (100%) diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 11a7d70c2..000000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -conda: - file: python/doc/environment.yml diff --git a/ballista/client/src/columnar_batch.rs b/ballista/client/src/columnar_batch.rs deleted file mode 100644 index 5e7fe89b0..000000000 --- a/ballista/client/src/columnar_batch.rs +++ /dev/null @@ -1,165 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::{collections::HashMap, sync::Arc}; - -use ballista_core::error::{ballista_error, Result}; - -use datafusion::arrow::{ - array::ArrayRef, - datatypes::{DataType, Schema}, - record_batch::RecordBatch, -}; -use datafusion::scalar::ScalarValue; - -pub type MaybeColumnarBatch = Result>; - -/// Batch of columnar data. -#[derive(Debug, Clone)] -pub struct ColumnarBatch { - schema: Arc, - columns: HashMap, -} - -impl ColumnarBatch { - pub fn from_arrow(batch: &RecordBatch) -> Self { - let columns = batch - .columns() - .iter() - .enumerate() - .map(|(i, array)| { - ( - batch.schema().field(i).name().clone(), - ColumnarValue::Columnar(array.clone()), - ) - }) - .collect(); - - Self { - schema: batch.schema(), - columns, - } - } - - pub fn from_values(values: &[ColumnarValue], schema: &Schema) -> Self { - let columns = schema - .fields() - .iter() - .enumerate() - .map(|(i, f)| (f.name().clone(), values[i].clone())) - .collect(); - - Self { - schema: Arc::new(schema.clone()), - columns, - } - } - - pub fn to_arrow(&self) -> Result { - let arrays = self - .schema - .fields() - .iter() - .map(|c| { - match self.column(c.name())? { - ColumnarValue::Columnar(array) => Ok(array.clone()), - ColumnarValue::Scalar(_, _) => { - // note that this can be implemented easily if needed - Err(ballista_error("Cannot convert scalar value to Arrow array")) - } - } - }) - .collect::>>()?; - - Ok(RecordBatch::try_new(self.schema.clone(), arrays)?) - } - - pub fn schema(&self) -> Arc { - self.schema.clone() - } - - pub fn num_columns(&self) -> usize { - self.columns.len() - } - - pub fn num_rows(&self) -> usize { - self.columns[self.schema.field(0).name()].len() - } - - pub fn column(&self, name: &str) -> Result<&ColumnarValue> { - Ok(&self.columns[name]) - } - - pub fn memory_size(&self) -> usize { - self.columns.values().map(|c| c.memory_size()).sum() - } -} - -/// A columnar value can either be a scalar value or an Arrow array. -#[derive(Debug, Clone)] -pub enum ColumnarValue { - Scalar(ScalarValue, usize), - Columnar(ArrayRef), -} - -impl ColumnarValue { - pub fn len(&self) -> usize { - match self { - ColumnarValue::Scalar(_, n) => *n, - ColumnarValue::Columnar(array) => array.len(), - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn data_type(&self) -> &DataType { - match self { - ColumnarValue::Columnar(array) => array.data_type(), - ColumnarValue::Scalar(value, _) => match value { - ScalarValue::UInt8(_) => &DataType::UInt8, - ScalarValue::UInt16(_) => &DataType::UInt16, - ScalarValue::UInt32(_) => &DataType::UInt32, - ScalarValue::UInt64(_) => &DataType::UInt64, - ScalarValue::Int8(_) => &DataType::Int8, - ScalarValue::Int16(_) => &DataType::Int16, - ScalarValue::Int32(_) => &DataType::Int32, - ScalarValue::Int64(_) => &DataType::Int64, - ScalarValue::Float32(_) => &DataType::Float32, - ScalarValue::Float64(_) => &DataType::Float64, - _ => unimplemented!(), - }, - } - } - - pub fn to_arrow(&self) -> Result { - match self { - ColumnarValue::Columnar(array) => Ok(array.clone()), - ColumnarValue::Scalar(value, n) => { - value.to_array_of_size(*n).map_err(|x| x.into()) - } - } - } - - pub fn memory_size(&self) -> usize { - match self { - ColumnarValue::Columnar(array) => array.get_array_memory_size(), - _ => 0, - } - } -} diff --git a/ballista/client/src/lib.rs b/ballista/client/src/lib.rs index 125278dcc..e61dfef28 100644 --- a/ballista/client/src/lib.rs +++ b/ballista/client/src/lib.rs @@ -17,6 +17,5 @@ #![doc = include_str!("../README.md")] -pub mod columnar_batch; pub mod context; pub mod prelude; diff --git a/examples/src/bin/dataframe.rs b/examples/examples/remote-dataframe.rs similarity index 100% rename from examples/src/bin/dataframe.rs rename to examples/examples/remote-dataframe.rs diff --git a/examples/src/bin/sql.rs b/examples/examples/remote-sql.rs similarity index 100% rename from examples/src/bin/sql.rs rename to examples/examples/remote-sql.rs From 37b4b0ea341044c23c551dae767799ec0863483a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Sat, 12 Oct 2024 15:37:58 +0100 Subject: [PATCH 5/5] Remove UI (#1072) --- .github/workflows/dev.yml | 3 +- .github/workflows/rust.yml | 46 +- CONTRIBUTING.md | 6 - README.md | 3 +- ballista/scheduler/ui/.dockerignore | 2 - ballista/scheduler/ui/.gitignore | 23 - ballista/scheduler/ui/README.md | 61 - ballista/scheduler/ui/index.d.ts | 18 - ballista/scheduler/ui/package.json | 60 - ballista/scheduler/ui/public/favicon.ico | Bin 3870 -> 0 bytes ballista/scheduler/ui/public/index.html | 62 - ballista/scheduler/ui/public/logo192.png | Bin 5347 -> 0 bytes ballista/scheduler/ui/public/logo512.png | Bin 9664 -> 0 bytes ballista/scheduler/ui/public/manifest.json | 25 - ballista/scheduler/ui/public/robots.txt | 20 - ballista/scheduler/ui/react-table-config.d.ts | 146 - ballista/scheduler/ui/src/App.css | 18 - ballista/scheduler/ui/src/App.test.tsx | 26 - ballista/scheduler/ui/src/App.tsx | 107 - .../scheduler/ui/src/components/DataTable.tsx | 197 - .../scheduler/ui/src/components/Empty.tsx | 36 - .../ui/src/components/ExecutorsList.tsx | 72 - .../scheduler/ui/src/components/Footer.tsx | 34 - .../scheduler/ui/src/components/Header.tsx | 87 - .../ui/src/components/JobStagesMetrics.tsx | 92 - .../ui/src/components/QueriesList.tsx | 267 - .../scheduler/ui/src/components/Summary.tsx | 91 - ballista/scheduler/ui/src/components/logo.svg | 25 - ballista/scheduler/ui/src/index.css | 32 - ballista/scheduler/ui/src/index.tsx | 39 - ballista/scheduler/ui/src/react-app-env.d.ts | 18 - ballista/scheduler/ui/src/reportWebVitals.ts | 32 - ballista/scheduler/ui/src/setupProxy.js | 32 - ballista/scheduler/ui/src/setupTests.ts | 22 - ballista/scheduler/ui/tsconfig.json | 28 - ballista/scheduler/ui/yarn.lock | 11020 ---------------- benchmarks/spark/README.md | 8 +- dev/docker/ballista-scheduler.Dockerfile | 8 +- dev/docker/ballista-standalone.Dockerfile | 8 +- dev/docker/builder-entrypoint.sh | 4 - dev/docker/nginx.conf | 53 - dev/docker/scheduler-entrypoint.sh | 2 - dev/docker/standalone-entrypoint.sh | 3 - docker-compose.yml | 1 - .../source/contributors-guide/architecture.md | 1 - .../contributors-guide/code-organization.md | 1 - .../user-guide/deployment/docker-compose.md | 2 - docs/source/user-guide/deployment/docker.md | 4 +- .../user-guide/deployment/kubernetes.md | 4 +- .../user-guide/deployment/quick-start.md | 10 - docs/source/user-guide/introduction.md | 4 - docs/source/user-guide/scheduler.md | 6 - docs/source/user-guide/tuning-guide.md | 2 +- 53 files changed, 12 insertions(+), 12859 deletions(-) delete mode 100644 ballista/scheduler/ui/.dockerignore delete mode 100644 ballista/scheduler/ui/.gitignore delete mode 100644 ballista/scheduler/ui/README.md delete mode 100644 ballista/scheduler/ui/index.d.ts delete mode 100644 ballista/scheduler/ui/package.json delete mode 100644 ballista/scheduler/ui/public/favicon.ico delete mode 100644 ballista/scheduler/ui/public/index.html delete mode 100644 ballista/scheduler/ui/public/logo192.png delete mode 100644 ballista/scheduler/ui/public/logo512.png delete mode 100644 ballista/scheduler/ui/public/manifest.json delete mode 100644 ballista/scheduler/ui/public/robots.txt delete mode 100644 ballista/scheduler/ui/react-table-config.d.ts delete mode 100644 ballista/scheduler/ui/src/App.css delete mode 100644 ballista/scheduler/ui/src/App.test.tsx delete mode 100644 ballista/scheduler/ui/src/App.tsx delete mode 100644 ballista/scheduler/ui/src/components/DataTable.tsx delete mode 100644 ballista/scheduler/ui/src/components/Empty.tsx delete mode 100644 ballista/scheduler/ui/src/components/ExecutorsList.tsx delete mode 100644 ballista/scheduler/ui/src/components/Footer.tsx delete mode 100644 ballista/scheduler/ui/src/components/Header.tsx delete mode 100644 ballista/scheduler/ui/src/components/JobStagesMetrics.tsx delete mode 100644 ballista/scheduler/ui/src/components/QueriesList.tsx delete mode 100644 ballista/scheduler/ui/src/components/Summary.tsx delete mode 100644 ballista/scheduler/ui/src/components/logo.svg delete mode 100644 ballista/scheduler/ui/src/index.css delete mode 100644 ballista/scheduler/ui/src/index.tsx delete mode 100644 ballista/scheduler/ui/src/react-app-env.d.ts delete mode 100644 ballista/scheduler/ui/src/reportWebVitals.ts delete mode 100644 ballista/scheduler/ui/src/setupProxy.js delete mode 100644 ballista/scheduler/ui/src/setupTests.ts delete mode 100644 ballista/scheduler/ui/tsconfig.json delete mode 100644 ballista/scheduler/ui/yarn.lock delete mode 100644 dev/docker/nginx.conf diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index c90948941..b71c5d984 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -69,6 +69,5 @@ jobs: '{ballista,docs}/**/*.md' \ '!ballista/CHANGELOG.md' \ README.md \ - CONTRIBUTING.md \ - 'ballista/**/*.{ts,tsx}' + CONTRIBUTING.md git diff --exit-code diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 454235617..db1b7fc92 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -72,44 +72,6 @@ jobs: /github/home/target/release/ballista-scheduler /github/home/target/release/ballista-executor - react-build: - name: React build - runs-on: ubuntu-latest # proprietary github image, not ubuntu:latest - container: - image: ubuntu:latest # actual ubuntu:latest that ubuntu publishes - env: - DEBIAN_FRONTEND: "noninteractive" - steps: - - uses: actions/checkout@v3 - - name: Install dependencies - run: | - cat /etc/*-release - apt-get -qq update - apt-get -qq upgrade - apt-get -qq install -y curl - curl -fsSL https://deb.nodesource.com/setup_18.x | bash - apt-get -qq update - apt-get -qq install -y nodejs - npm install -g yarn - which node - which npm - which yarn - node --version - npm --version - yarn --version - - name: Run yarn build - run: | - cd ballista/scheduler/ui - pwd - yarn install - yarn build - - name: Save artifacts - uses: actions/upload-artifact@v4 - with: - name: react-artifacts - path: | - ballista/scheduler/ui/build - # test the crate linux-test: name: Test Workspace on AMD64 Rust ${{ matrix.rust }} @@ -338,7 +300,7 @@ jobs: docker: name: Docker - needs: [linux-build-lib, react-build] + needs: [linux-build-lib] runs-on: ubuntu-latest permissions: contents: read @@ -350,11 +312,6 @@ jobs: with: name: rust-artifacts path: target/release - - name: Restore react artifacts - uses: actions/download-artifact@v4 - with: - name: react-artifacts - path: ballista/scheduler/ui/build - name: Build and push Docker image run: | echo "github user is $DOCKER_USER" @@ -468,7 +425,6 @@ jobs: env: CARGO_HOME: "/github/home/.cargo" CARGO_TARGET_DIR: "/github/home/target" - # Coverage job was failing. https://github.com/apache/arrow-datafusion/issues/590 tracks re-instating it # coverage: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66de725ac..9a77fef38 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,8 +41,6 @@ This section describes how you can get started with Ballista development. Ballista contains components implemented in the following programming languages: - Rust (Scheduler and Executor processes, Client library) -- Python (Python bindings) -- Javascript (Scheduler Web UI) ### Rust Environment @@ -95,10 +93,6 @@ for the executor becomes `BALLISTA_EXECUTOR_SCHEDULER_HOST` Refer to the instructions in the Python Bindings [README](./python/README.md) -### Javascript Environment - -Refer to the instructions in the Scheduler Web UI [README](./ballista/scheduler/ui/README.md) - ## Integration Tests The integration tests can be executed by running the following command from the root of the repository. diff --git a/README.md b/README.md index 4b149f5d1..80c108178 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ See the [architecture guide](docs/source/contributors-guide/architecture.md) for - DataFrame and SQL APIs available from Python and Rust. - Clients can connect to a Ballista cluster using [Flight SQL][flight-sql]. - JDBC support via Arrow Flight SQL JDBC Driver -- Scheduler web interface and REST UI for monitoring query progress and viewing query plans and metrics. +- Scheduler REST UI for monitoring query progress and viewing query plans and metrics. - Support for Docker, Docker Compose, and Kubernetes deployment, as well as manual deployment on bare metal. ## Performance @@ -105,5 +105,4 @@ Please see the [Contribution Guide](CONTRIBUTING.md) for information about contr [datafusion]: https://github.com/apache/arrow-datafusion [flight]: https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/ [flight-sql]: https://arrow.apache.org/blog/2022/02/16/introducing-arrow-flight-sql/ -[ballista-talk]: https://www.youtube.com/watch?v=ZZHQaOap9pQ [user-guide]: https://datafusion.apache.org/ballista/ diff --git a/ballista/scheduler/ui/.dockerignore b/ballista/scheduler/ui/.dockerignore deleted file mode 100644 index dd87e2d73..000000000 --- a/ballista/scheduler/ui/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules -build diff --git a/ballista/scheduler/ui/.gitignore b/ballista/scheduler/ui/.gitignore deleted file mode 100644 index 15a75a078..000000000 --- a/ballista/scheduler/ui/.gitignore +++ /dev/null @@ -1,23 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -node_modules/ -/.pnp -.pnp.js - -# testing -/coverage - -# production -build/ - -# misc -.DS_Store -.env.local -.env.development.local -.env.test.local -.env.production.local - -npm-debug.log* -yarn-debug.log* -yarn-error.log* diff --git a/ballista/scheduler/ui/README.md b/ballista/scheduler/ui/README.md deleted file mode 100644 index 4e72825c5..000000000 --- a/ballista/scheduler/ui/README.md +++ /dev/null @@ -1,61 +0,0 @@ - - -# Ballista UI - -## Start project from source - -### Run scheduler/executor - -First, run scheduler from project: - -```shell -$ cd ballista/scheduler -$ RUST_LOG=info cargo run --release -... - Finished release [optimized] target(s) in 11.92s - Running `/path-to-project/target/release/ballista-scheduler` -``` - -and run executor in new terminal: - -```shell -$ cd ballista/executor -$ RUST_LOG=info cargo run --release - Finished release [optimized] target(s) in 0.09s - Running `/path-to-project/target/release/ballista-executor` -``` - -### Run Client project - -```shell -$ cd ballista/scheduler/ui -$ yarn -Resolving packages... -$ yarn start -Starting the development server... -``` - -Now access to http://localhost:3000/ - -**NOTE**: If you get an error when running the `yarn start` command above, make sure -to use the `lts` version of npm. - -You can install it by running: `npm install --lts`. This should fix any errors that -may occur when trying to start the Ballista UI project. diff --git a/ballista/scheduler/ui/index.d.ts b/ballista/scheduler/ui/index.d.ts deleted file mode 100644 index be0f38ba0..000000000 --- a/ballista/scheduler/ui/index.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -declare module "@chakra-ui/icons"; diff --git a/ballista/scheduler/ui/package.json b/ballista/scheduler/ui/package.json deleted file mode 100644 index 1bf27a8a8..000000000 --- a/ballista/scheduler/ui/package.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "name": "scheduler-ui", - "version": "0.1.0", - "private": true, - "dependencies": { - "@chakra-ui/icons": "^2.1.1", - "@chakra-ui/react": "^2.8.2", - "@emotion/react": "^11.11.3", - "@emotion/styled": "^11.11.0", - "@testing-library/jest-dom": "^6.2.1", - "@testing-library/react": "^14.1.2", - "@testing-library/user-event": "^14.5.2", - "@types/jest": "^29.5.11", - "@types/node": "^20.11.5", - "@types/react": "^18.2.48", - "@types/react-dom": "^18.2.18", - "framer-motion": "^11.0.2", - "js-file-download": "^0.4.12", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "react-icons": "^5.0.1", - "react-inlinesvg": "^4.1.1", - "react-router-dom": "^6.21.3", - "react-scripts": "5.0.1", - "react-table": "^7.8.0", - "react-timeago": "^7.2.0", - "typescript": "^5.3.3", - "web-vitals": "^3.5.1" - }, - "scripts": { - "start": "react-scripts --openssl-legacy-provider start", - "build": "react-scripts --openssl-legacy-provider build", - "test": "react-scripts test", - "eject": "react-scripts eject" - }, - "eslintConfig": { - "extends": [ - "react-app", - "react-app/jest" - ] - }, - "browserslist": { - "production": [ - ">0.2%", - "not dead", - "not op_mini all" - ], - "development": [ - "last 1 chrome version", - "last 1 firefox version", - "last 1 safari version" - ] - }, - "devDependencies": { - "@types/react-table": "^7.7.19", - "@types/react-timeago": "^4.1.7", - "prettier": "^3.2.4", - "http-proxy-middleware": "^2.0.6" - } -} diff --git a/ballista/scheduler/ui/public/favicon.ico b/ballista/scheduler/ui/public/favicon.ico deleted file mode 100644 index a11777cc471a4344702741ab1c8a588998b1311a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3870 zcma);c{J4h9>;%nil|2-o+rCuEF-(I%-F}ijC~o(k~HKAkr0)!FCj~d>`RtpD?8b; zXOC1OD!V*IsqUwzbMF1)-gEDD=A573Z-&G7^LoAC9|WO7Xc0Cx1g^Zu0u_SjAPB3vGa^W|sj)80f#V0@M_CAZTIO(t--xg= z!sii`1giyH7EKL_+Wi0ab<)&E_0KD!3Rp2^HNB*K2@PHCs4PWSA32*-^7d{9nH2_E zmC{C*N*)(vEF1_aMamw2A{ZH5aIDqiabnFdJ|y0%aS|64E$`s2ccV~3lR!u<){eS` z#^Mx6o(iP1Ix%4dv`t@!&Za-K@mTm#vadc{0aWDV*_%EiGK7qMC_(`exc>-$Gb9~W!w_^{*pYRm~G zBN{nA;cm^w$VWg1O^^<6vY`1XCD|s_zv*g*5&V#wv&s#h$xlUilPe4U@I&UXZbL z0)%9Uj&@yd03n;!7do+bfixH^FeZ-Ema}s;DQX2gY+7g0s(9;`8GyvPY1*vxiF&|w z>!vA~GA<~JUqH}d;DfBSi^IT*#lrzXl$fNpq0_T1tA+`A$1?(gLb?e#0>UELvljtQ zK+*74m0jn&)5yk8mLBv;=@}c{t0ztT<v;Avck$S6D`Z)^c0(jiwKhQsn|LDRY&w(Fmi91I7H6S;b0XM{e zXp0~(T@k_r-!jkLwd1_Vre^v$G4|kh4}=Gi?$AaJ)3I+^m|Zyj#*?Kp@w(lQdJZf4 z#|IJW5z+S^e9@(6hW6N~{pj8|NO*>1)E=%?nNUAkmv~OY&ZV;m-%?pQ_11)hAr0oAwILrlsGawpxx4D43J&K=n+p3WLnlDsQ$b(9+4 z?mO^hmV^F8MV{4Lx>(Q=aHhQ1){0d*(e&s%G=i5rq3;t{JC zmgbn5Nkl)t@fPH$v;af26lyhH!k+#}_&aBK4baYPbZy$5aFx4}ka&qxl z$=Rh$W;U)>-=S-0=?7FH9dUAd2(q#4TCAHky!$^~;Dz^j|8_wuKc*YzfdAht@Q&ror?91Dm!N03=4=O!a)I*0q~p0g$Fm$pmr$ zb;wD;STDIi$@M%y1>p&_>%?UP($15gou_ue1u0!4(%81;qcIW8NyxFEvXpiJ|H4wz z*mFT(qVx1FKufG11hByuX%lPk4t#WZ{>8ka2efjY`~;AL6vWyQKpJun2nRiZYDij$ zP>4jQXPaP$UC$yIVgGa)jDV;F0l^n(V=HMRB5)20V7&r$jmk{UUIe zVjKroK}JAbD>B`2cwNQ&GDLx8{pg`7hbA~grk|W6LgiZ`8y`{Iq0i>t!3p2}MS6S+ zO_ruKyAElt)rdS>CtF7j{&6rP-#c=7evGMt7B6`7HG|-(WL`bDUAjyn+k$mx$CH;q2Dz4x;cPP$hW=`pFfLO)!jaCL@V2+F)So3}vg|%O*^T1j>C2lx zsURO-zIJC$^$g2byVbRIo^w>UxK}74^TqUiRR#7s_X$e)$6iYG1(PcW7un-va-S&u zHk9-6Zn&>T==A)lM^D~bk{&rFzCi35>UR!ZjQkdSiNX*-;l4z9j*7|q`TBl~Au`5& z+c)*8?#-tgUR$Zd%Q3bs96w6k7q@#tUn`5rj+r@_sAVVLqco|6O{ILX&U-&-cbVa3 zY?ngHR@%l{;`ri%H*0EhBWrGjv!LE4db?HEWb5mu*t@{kv|XwK8?npOshmzf=vZA@ zVSN9sL~!sn?r(AK)Q7Jk2(|M67Uy3I{eRy z_l&Y@A>;vjkWN5I2xvFFTLX0i+`{qz7C_@bo`ZUzDugfq4+>a3?1v%)O+YTd6@Ul7 zAfLfm=nhZ`)P~&v90$&UcF+yXm9sq!qCx3^9gzIcO|Y(js^Fj)Rvq>nQAHI92ap=P z10A4@prk+AGWCb`2)dQYFuR$|H6iDE8p}9a?#nV2}LBCoCf(Xi2@szia7#gY>b|l!-U`c}@ zLdhvQjc!BdLJvYvzzzngnw51yRYCqh4}$oRCy-z|v3Hc*d|?^Wj=l~18*E~*cR_kU z{XsxM1i{V*4GujHQ3DBpl2w4FgFR48Nma@HPgnyKoIEY-MqmMeY=I<%oG~l!f<+FN z1ZY^;10j4M4#HYXP zw5eJpA_y(>uLQ~OucgxDLuf}fVs272FaMxhn4xnDGIyLXnw>Xsd^J8XhcWIwIoQ9} z%FoSJTAGW(SRGwJwb=@pY7r$uQRK3Zd~XbxU)ts!4XsJrCycrWSI?e!IqwqIR8+Jh zlRjZ`UO1I!BtJR_2~7AbkbSm%XQqxEPkz6BTGWx8e}nQ=w7bZ|eVP4?*Tb!$(R)iC z9)&%bS*u(lXqzitAN)Oo=&Ytn>%Hzjc<5liuPi>zC_nw;Z0AE3Y$Jao_Q90R-gl~5 z_xAb2J%eArrC1CN4G$}-zVvCqF1;H;abAu6G*+PDHSYFx@Tdbfox*uEd3}BUyYY-l zTfEsOqsi#f9^FoLO;ChK<554qkri&Av~SIM*{fEYRE?vH7pTAOmu2pz3X?Wn*!ROX ztd54huAk&mFBemMooL33RV-*1f0Q3_(7hl$<#*|WF9P!;r;4_+X~k~uKEqdzZ$5Al zV63XN@)j$FN#cCD;ek1R#l zv%pGrhB~KWgoCj%GT?%{@@o(AJGt*PG#l3i>lhmb_twKH^EYvacVY-6bsCl5*^~L0 zonm@lk2UvvTKr2RS%}T>^~EYqdL1q4nD%0n&Xqr^cK^`J5W;lRRB^R-O8b&HENO||mo0xaD+S=I8RTlIfVgqN@SXDr2&-)we--K7w= zJVU8?Z+7k9dy;s;^gDkQa`0nz6N{T?(A&Iz)2!DEecLyRa&FI!id#5Z7B*O2=PsR0 zEvc|8{NS^)!d)MDX(97Xw}m&kEO@5jqRaDZ!+%`wYOI<23q|&js`&o4xvjP7D_xv@ z5hEwpsp{HezI9!~6O{~)lLR@oF7?J7i>1|5a~UuoN=q&6N}EJPV_GD`&M*v8Y`^2j zKII*d_@Fi$+i*YEW+Hbzn{iQk~yP z>7N{S4)r*!NwQ`(qcN#8SRQsNK6>{)X12nbF`*7#ecO7I)Q$uZsV+xS4E7aUn+U(K baj7?x%VD!5Cxk2YbYLNVeiXvvpMCWYo=by@ diff --git a/ballista/scheduler/ui/public/index.html b/ballista/scheduler/ui/public/index.html deleted file mode 100644 index d902333f0..000000000 --- a/ballista/scheduler/ui/public/index.html +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - Ballista UI - - - -
- - - diff --git a/ballista/scheduler/ui/public/logo192.png b/ballista/scheduler/ui/public/logo192.png deleted file mode 100644 index fc44b0a3796c0e0a64c3d858ca038bd4570465d9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5347 zcmZWtbyO6NvR-oO24RV%BvuJ&=?+<7=`LvyB&A_#M7mSDYw1v6DJkiYl9XjT!%$dLEBTQ8R9|wd3008in6lFF3GV-6mLi?MoP_y~}QUnaDCHI#t z7w^m$@6DI)|C8_jrT?q=f8D?0AM?L)Z}xAo^e^W>t$*Y0KlT5=@bBjT9kxb%-KNdk zeOS1tKO#ChhG7%{ApNBzE2ZVNcxbrin#E1TiAw#BlUhXllzhN$qWez5l;h+t^q#Eav8PhR2|T}y5kkflaK`ba-eoE+Z2q@o6P$)=&` z+(8}+-McnNO>e#$Rr{32ngsZIAX>GH??tqgwUuUz6kjns|LjsB37zUEWd|(&O!)DY zQLrq%Y>)Y8G`yYbYCx&aVHi@-vZ3|ebG!f$sTQqMgi0hWRJ^Wc+Ibv!udh_r%2|U) zPi|E^PK?UE!>_4`f`1k4hqqj_$+d!EB_#IYt;f9)fBOumGNyglU(ofY`yHq4Y?B%- zp&G!MRY<~ajTgIHErMe(Z8JG*;D-PJhd@RX@QatggM7+G(Lz8eZ;73)72Hfx5KDOE zkT(m}i2;@X2AT5fW?qVp?@WgN$aT+f_6eo?IsLh;jscNRp|8H}Z9p_UBO^SJXpZew zEK8fz|0Th%(Wr|KZBGTM4yxkA5CFdAj8=QSrT$fKW#tweUFqr0TZ9D~a5lF{)%-tTGMK^2tz(y2v$i%V8XAxIywrZCp=)83p(zIk6@S5AWl|Oa2hF`~~^W zI;KeOSkw1O#TiQ8;U7OPXjZM|KrnN}9arP)m0v$c|L)lF`j_rpG(zW1Qjv$=^|p*f z>)Na{D&>n`jOWMwB^TM}slgTEcjxTlUby89j1)|6ydRfWERn3|7Zd2&e7?!K&5G$x z`5U3uFtn4~SZq|LjFVrz$3iln-+ucY4q$BC{CSm7Xe5c1J<=%Oagztj{ifpaZk_bQ z9Sb-LaQMKp-qJA*bP6DzgE3`}*i1o3GKmo2pn@dj0;He}F=BgINo};6gQF8!n0ULZ zL>kC0nPSFzlcB7p41doao2F7%6IUTi_+!L`MM4o*#Y#0v~WiO8uSeAUNp=vA2KaR&=jNR2iVwG>7t%sG2x_~yXzY)7K& zk3p+O0AFZ1eu^T3s};B%6TpJ6h-Y%B^*zT&SN7C=N;g|#dGIVMSOru3iv^SvO>h4M=t-N1GSLLDqVTcgurco6)3&XpU!FP6Hlrmj}f$ zp95;b)>M~`kxuZF3r~a!rMf4|&1=uMG$;h^g=Kl;H&Np-(pFT9FF@++MMEx3RBsK?AU0fPk-#mdR)Wdkj)`>ZMl#^<80kM87VvsI3r_c@_vX=fdQ`_9-d(xiI z4K;1y1TiPj_RPh*SpDI7U~^QQ?%0&!$Sh#?x_@;ag)P}ZkAik{_WPB4rHyW#%>|Gs zdbhyt=qQPA7`?h2_8T;-E6HI#im9K>au*(j4;kzwMSLgo6u*}-K`$_Gzgu&XE)udQ zmQ72^eZd|vzI)~!20JV-v-T|<4@7ruqrj|o4=JJPlybwMg;M$Ud7>h6g()CT@wXm` zbq=A(t;RJ^{Xxi*Ff~!|3!-l_PS{AyNAU~t{h;(N(PXMEf^R(B+ZVX3 z8y0;0A8hJYp@g+c*`>eTA|3Tgv9U8#BDTO9@a@gVMDxr(fVaEqL1tl?md{v^j8aUv zm&%PX4^|rX|?E4^CkplWWNv*OKM>DxPa z!RJ)U^0-WJMi)Ksc!^ixOtw^egoAZZ2Cg;X7(5xZG7yL_;UJ#yp*ZD-;I^Z9qkP`} zwCTs0*%rIVF1sgLervtnUo&brwz?6?PXRuOCS*JI-WL6GKy7-~yi0giTEMmDs_-UX zo=+nFrW_EfTg>oY72_4Z0*uG>MnXP=c0VpT&*|rvv1iStW;*^={rP1y?Hv+6R6bxFMkxpWkJ>m7Ba{>zc_q zEefC3jsXdyS5??Mz7IET$Kft|EMNJIv7Ny8ZOcKnzf`K5Cd)&`-fTY#W&jnV0l2vt z?Gqhic}l}mCv1yUEy$%DP}4AN;36$=7aNI^*AzV(eYGeJ(Px-j<^gSDp5dBAv2#?; zcMXv#aj>%;MiG^q^$0MSg-(uTl!xm49dH!{X0){Ew7ThWV~Gtj7h%ZD zVN-R-^7Cf0VH!8O)uUHPL2mO2tmE*cecwQv_5CzWeh)ykX8r5Hi`ehYo)d{Jnh&3p z9ndXT$OW51#H5cFKa76c<%nNkP~FU93b5h-|Cb}ScHs@4Q#|}byWg;KDMJ#|l zE=MKD*F@HDBcX@~QJH%56eh~jfPO-uKm}~t7VkHxHT;)4sd+?Wc4* z>CyR*{w@4(gnYRdFq=^(#-ytb^5ESD?x<0Skhb%Pt?npNW1m+Nv`tr9+qN<3H1f<% zZvNEqyK5FgPsQ`QIu9P0x_}wJR~^CotL|n zk?dn;tLRw9jJTur4uWoX6iMm914f0AJfB@C74a;_qRrAP4E7l890P&{v<}>_&GLrW z)klculcg`?zJO~4;BBAa=POU%aN|pmZJn2{hA!d!*lwO%YSIzv8bTJ}=nhC^n}g(ld^rn#kq9Z3)z`k9lvV>y#!F4e{5c$tnr9M{V)0m(Z< z#88vX6-AW7T2UUwW`g<;8I$Jb!R%z@rCcGT)-2k7&x9kZZT66}Ztid~6t0jKb&9mm zpa}LCb`bz`{MzpZR#E*QuBiZXI#<`5qxx=&LMr-UUf~@dRk}YI2hbMsAMWOmDzYtm zjof16D=mc`^B$+_bCG$$@R0t;e?~UkF?7<(vkb70*EQB1rfUWXh$j)R2)+dNAH5%R zEBs^?N;UMdy}V};59Gu#0$q53$}|+q7CIGg_w_WlvE}AdqoS<7DY1LWS9?TrfmcvT zaypmplwn=P4;a8-%l^e?f`OpGb}%(_mFsL&GywhyN(-VROj`4~V~9bGv%UhcA|YW% zs{;nh@aDX11y^HOFXB$a7#Sr3cEtNd4eLm@Y#fc&j)TGvbbMwze zXtekX_wJqxe4NhuW$r}cNy|L{V=t#$%SuWEW)YZTH|!iT79k#?632OFse{+BT_gau zJwQcbH{b}dzKO?^dV&3nTILYlGw{27UJ72ZN){BILd_HV_s$WfI2DC<9LIHFmtyw? zQ;?MuK7g%Ym+4e^W#5}WDLpko%jPOC=aN)3!=8)s#Rnercak&b3ESRX3z{xfKBF8L z5%CGkFmGO@x?_mPGlpEej!3!AMddChabyf~nJNZxx!D&{@xEb!TDyvqSj%Y5@A{}9 zRzoBn0?x}=krh{ok3Nn%e)#~uh;6jpezhA)ySb^b#E>73e*frBFu6IZ^D7Ii&rsiU z%jzygxT-n*joJpY4o&8UXr2s%j^Q{?e-voloX`4DQyEK+DmrZh8A$)iWL#NO9+Y@!sO2f@rI!@jN@>HOA< z?q2l{^%mY*PNx2FoX+A7X3N}(RV$B`g&N=e0uvAvEN1W^{*W?zT1i#fxuw10%~))J zjx#gxoVlXREWZf4hRkgdHx5V_S*;p-y%JtGgQ4}lnA~MBz-AFdxUxU1RIT$`sal|X zPB6sEVRjGbXIP0U+?rT|y5+ev&OMX*5C$n2SBPZr`jqzrmpVrNciR0e*Wm?fK6DY& zl(XQZ60yWXV-|Ps!A{EF;=_z(YAF=T(-MkJXUoX zI{UMQDAV2}Ya?EisdEW;@pE6dt;j0fg5oT2dxCi{wqWJ<)|SR6fxX~5CzblPGr8cb zUBVJ2CQd~3L?7yfTpLNbt)He1D>*KXI^GK%<`bq^cUq$Q@uJifG>p3LU(!H=C)aEL zenk7pVg}0{dKU}&l)Y2Y2eFMdS(JS0}oZUuVaf2+K*YFNGHB`^YGcIpnBlMhO7d4@vV zv(@N}(k#REdul8~fP+^F@ky*wt@~&|(&&meNO>rKDEnB{ykAZ}k>e@lad7to>Ao$B zz<1(L=#J*u4_LB=8w+*{KFK^u00NAmeNN7pr+Pf+N*Zl^dO{LM-hMHyP6N!~`24jd zXYP|Ze;dRXKdF2iJG$U{k=S86l@pytLx}$JFFs8e)*Vi?aVBtGJ3JZUj!~c{(rw5>vuRF$`^p!P8w1B=O!skwkO5yd4_XuG^QVF z`-r5K7(IPSiKQ2|U9+`@Js!g6sfJwAHVd|s?|mnC*q zp|B|z)(8+mxXyxQ{8Pg3F4|tdpgZZSoU4P&9I8)nHo1@)9_9u&NcT^FI)6|hsAZFk zZ+arl&@*>RXBf-OZxhZerOr&dN5LW9@gV=oGFbK*J+m#R-|e6(Loz(;g@T^*oO)0R zN`N=X46b{7yk5FZGr#5&n1!-@j@g02g|X>MOpF3#IjZ_4wg{dX+G9eqS+Es9@6nC7 zD9$NuVJI}6ZlwtUm5cCAiYv0(Yi{%eH+}t)!E^>^KxB5^L~a`4%1~5q6h>d;paC9c zTj0wTCKrhWf+F#5>EgX`sl%POl?oyCq0(w0xoL?L%)|Q7d|Hl92rUYAU#lc**I&^6p=4lNQPa0 znQ|A~i0ip@`B=FW-Q;zh?-wF;Wl5!+q3GXDu-x&}$gUO)NoO7^$BeEIrd~1Dh{Tr` z8s<(Bn@gZ(mkIGnmYh_ehXnq78QL$pNDi)|QcT*|GtS%nz1uKE+E{7jdEBp%h0}%r zD2|KmYGiPa4;md-t_m5YDz#c*oV_FqXd85d@eub?9N61QuYcb3CnVWpM(D-^|CmkL z(F}L&N7qhL2PCq)fRh}XO@U`Yn<?TNGR4L(mF7#4u29{i~@k;pLsgl({YW5`Mo+p=zZn3L*4{JU;++dG9 X@eDJUQo;Ye2mwlRs?y0|+_a0zY+Zo%Dkae}+MySoIppb75o?vUW_?)>@g{U2`ERQIXV zeY$JrWnMZ$QC<=ii4X|@0H8`si75jB(ElJb00HAB%>SlLR{!zO|C9P3zxw_U8?1d8uRZ=({Ga4shyN}3 zAK}WA(ds|``G4jA)9}Bt2Hy0+f3rV1E6b|@?hpGA=PI&r8)ah|)I2s(P5Ic*Ndhn^ z*T&j@gbCTv7+8rpYbR^Ty}1AY)YH;p!m948r#%7x^Z@_-w{pDl|1S4`EM3n_PaXvK z1JF)E3qy$qTj5Xs{jU9k=y%SQ0>8E$;x?p9ayU0bZZeo{5Z@&FKX>}s!0+^>C^D#z z>xsCPvxD3Z=dP}TTOSJhNTPyVt14VCQ9MQFN`rn!c&_p?&4<5_PGm4a;WS&1(!qKE z_H$;dDdiPQ!F_gsN`2>`X}$I=B;={R8%L~`>RyKcS$72ai$!2>d(YkciA^J0@X%G4 z4cu!%Ps~2JuJ8ex`&;Fa0NQOq_nDZ&X;^A=oc1&f#3P1(!5il>6?uK4QpEG8z0Rhu zvBJ+A9RV?z%v?!$=(vcH?*;vRs*+PPbOQ3cdPr5=tOcLqmfx@#hOqX0iN)wTTO21jH<>jpmwRIAGw7`a|sl?9y9zRBh>(_%| zF?h|P7}~RKj?HR+q|4U`CjRmV-$mLW>MScKnNXiv{vD3&2@*u)-6P@h0A`eeZ7}71 zK(w%@R<4lLt`O7fs1E)$5iGb~fPfJ?WxhY7c3Q>T-w#wT&zW522pH-B%r5v#5y^CF zcC30Se|`D2mY$hAlIULL%-PNXgbbpRHgn<&X3N9W!@BUk@9g*P5mz-YnZBb*-$zMM z7Qq}ic0mR8n{^L|=+diODdV}Q!gwr?y+2m=3HWwMq4z)DqYVg0J~^}-%7rMR@S1;9 z7GFj6K}i32X;3*$SmzB&HW{PJ55kT+EI#SsZf}bD7nW^Haf}_gXciYKX{QBxIPSx2Ma? zHQqgzZq!_{&zg{yxqv3xq8YV+`S}F6A>Gtl39_m;K4dA{pP$BW0oIXJ>jEQ!2V3A2 zdpoTxG&V=(?^q?ZTj2ZUpDUdMb)T?E$}CI>r@}PFPWD9@*%V6;4Ag>D#h>!s)=$0R zRXvdkZ%|c}ubej`jl?cS$onl9Tw52rBKT)kgyw~Xy%z62Lr%V6Y=f?2)J|bZJ5(Wx zmji`O;_B+*X@qe-#~`HFP<{8$w@z4@&`q^Q-Zk8JG3>WalhnW1cvnoVw>*R@c&|o8 zZ%w!{Z+MHeZ*OE4v*otkZqz11*s!#s^Gq>+o`8Z5 z^i-qzJLJh9!W-;SmFkR8HEZJWiXk$40i6)7 zZpr=k2lp}SasbM*Nbn3j$sn0;rUI;%EDbi7T1ZI4qL6PNNM2Y%6{LMIKW+FY_yF3) zSKQ2QSujzNMSL2r&bYs`|i2Dnn z=>}c0>a}>|uT!IiMOA~pVT~R@bGlm}Edf}Kq0?*Af6#mW9f9!}RjW7om0c9Qlp;yK z)=XQs(|6GCadQbWIhYF=rf{Y)sj%^Id-ARO0=O^Ad;Ph+ z0?$eE1xhH?{T$QI>0JP75`r)U_$#%K1^BQ8z#uciKf(C701&RyLQWBUp*Q7eyn76} z6JHpC9}R$J#(R0cDCkXoFSp;j6{x{b&0yE@P7{;pCEpKjS(+1RQy38`=&Yxo%F=3y zCPeefABp34U-s?WmU#JJw23dcC{sPPFc2#J$ZgEN%zod}J~8dLm*fx9f6SpO zn^Ww3bt9-r0XaT2a@Wpw;C23XM}7_14#%QpubrIw5aZtP+CqIFmsG4`Cm6rfxl9n5 z7=r2C-+lM2AB9X0T_`?EW&Byv&K?HS4QLoylJ|OAF z`8atBNTzJ&AQ!>sOo$?^0xj~D(;kS$`9zbEGd>f6r`NC3X`tX)sWgWUUOQ7w=$TO&*j;=u%25ay-%>3@81tGe^_z*C7pb9y*Ed^H3t$BIKH2o+olp#$q;)_ zfpjCb_^VFg5fU~K)nf*d*r@BCC>UZ!0&b?AGk_jTPXaSnCuW110wjHPPe^9R^;jo3 zwvzTl)C`Zl5}O2}3lec=hZ*$JnkW#7enKKc)(pM${_$9Hc=Sr_A9Biwe*Y=T?~1CK z6eZ9uPICjy-sMGbZl$yQmpB&`ouS8v{58__t0$JP%i3R&%QR3ianbZqDs<2#5FdN@n5bCn^ZtH992~5k(eA|8|@G9u`wdn7bnpg|@{m z^d6Y`*$Zf2Xr&|g%sai#5}Syvv(>Jnx&EM7-|Jr7!M~zdAyjt*xl;OLhvW-a%H1m0 z*x5*nb=R5u><7lyVpNAR?q@1U59 zO+)QWwL8t zyip?u_nI+K$uh{y)~}qj?(w0&=SE^8`_WMM zTybjG=999h38Yes7}-4*LJ7H)UE8{mE(6;8voE+TYY%33A>S6`G_95^5QHNTo_;Ao ztIQIZ_}49%{8|=O;isBZ?=7kfdF8_@azfoTd+hEJKWE!)$)N%HIe2cplaK`ry#=pV z0q{9w-`i0h@!R8K3GC{ivt{70IWG`EP|(1g7i_Q<>aEAT{5(yD z=!O?kq61VegV+st@XCw475j6vS)_z@efuqQgHQR1T4;|-#OLZNQJPV4k$AX1Uk8Lm z{N*b*ia=I+MB}kWpupJ~>!C@xEN#Wa7V+7{m4j8c?)ChV=D?o~sjT?0C_AQ7B-vxqX30s0I_`2$in86#`mAsT-w?j{&AL@B3$;P z31G4(lV|b}uSDCIrjk+M1R!X7s4Aabn<)zpgT}#gE|mIvV38^ODy@<&yflpCwS#fRf9ZX3lPV_?8@C5)A;T zqmouFLFk;qIs4rA=hh=GL~sCFsXHsqO6_y~*AFt939UYVBSx1s(=Kb&5;j7cSowdE;7()CC2|-i9Zz+_BIw8#ll~-tyH?F3{%`QCsYa*b#s*9iCc`1P1oC26?`g<9))EJ3%xz+O!B3 zZ7$j~To)C@PquR>a1+Dh>-a%IvH_Y7^ys|4o?E%3`I&ADXfC8++hAdZfzIT#%C+Jz z1lU~K_vAm0m8Qk}K$F>|>RPK%<1SI0(G+8q~H zAsjezyP+u!Se4q3GW)`h`NPSRlMoBjCzNPesWJwVTY!o@G8=(6I%4XHGaSiS3MEBK zhgGFv6Jc>L$4jVE!I?TQuwvz_%CyO!bLh94nqK11C2W$*aa2ueGopG8DnBICVUORP zgytv#)49fVXDaR$SukloYC3u7#5H)}1K21=?DKj^U)8G;MS)&Op)g^zR2($<>C*zW z;X7`hLxiIO#J`ANdyAOJle4V%ppa*(+0i3w;8i*BA_;u8gOO6)MY`ueq7stBMJTB; z-a0R>hT*}>z|Gg}@^zDL1MrH+2hsR8 zHc}*9IvuQC^Ju)^#Y{fOr(96rQNPNhxc;mH@W*m206>Lo<*SaaH?~8zg&f&%YiOEG zGiz?*CP>Bci}!WiS=zj#K5I}>DtpregpP_tfZtPa(N<%vo^#WCQ5BTv0vr%Z{)0q+ z)RbfHktUm|lg&U3YM%lMUM(fu}i#kjX9h>GYctkx9Mt_8{@s%!K_EI zScgwy6%_fR?CGJQtmgNAj^h9B#zmaMDWgH55pGuY1Gv7D z;8Psm(vEPiwn#MgJYu4Ty9D|h!?Rj0ddE|&L3S{IP%H4^N!m`60ZwZw^;eg4sk6K{ ziA^`Sbl_4~f&Oo%n;8Ye(tiAdlZKI!Z=|j$5hS|D$bDJ}p{gh$KN&JZYLUjv4h{NY zBJ>X9z!xfDGY z+oh_Z&_e#Q(-}>ssZfm=j$D&4W4FNy&-kAO1~#3Im;F)Nwe{(*75(p=P^VI?X0GFakfh+X-px4a%Uw@fSbmp9hM1_~R>?Z8+ ziy|e9>8V*`OP}4x5JjdWp}7eX;lVxp5qS}0YZek;SNmm7tEeSF*-dI)6U-A%m6YvCgM(}_=k#a6o^%-K4{`B1+}O4x zztDT%hVb;v#?j`lTvlFQ3aV#zkX=7;YFLS$uIzb0E3lozs5`Xy zi~vF+%{z9uLjKvKPhP%x5f~7-Gj+%5N`%^=yk*Qn{`> z;xj&ROY6g`iy2a@{O)V(jk&8#hHACVDXey5a+KDod_Z&}kHM}xt7}Md@pil{2x7E~ zL$k^d2@Ec2XskjrN+IILw;#7((abu;OJii&v3?60x>d_Ma(onIPtcVnX@ELF0aL?T zSmWiL3(dOFkt!x=1O!_0n(cAzZW+3nHJ{2S>tgSK?~cFha^y(l@-Mr2W$%MN{#af8J;V*>hdq!gx=d0h$T7l}>91Wh07)9CTX zh2_ZdQCyFOQ)l(}gft0UZG`Sh2`x-w`5vC2UD}lZs*5 zG76$akzn}Xi))L3oGJ75#pcN=cX3!=57$Ha=hQ2^lwdyU#a}4JJOz6ddR%zae%#4& za)bFj)z=YQela(F#Y|Q#dp}PJghITwXouVaMq$BM?K%cXn9^Y@g43$=O)F&ZlOUom zJiad#dea;-eywBA@e&D6Pdso1?2^(pXiN91?jvcaUyYoKUmvl5G9e$W!okWe*@a<^ z8cQQ6cNSf+UPDx%?_G4aIiybZHHagF{;IcD(dPO!#=u zWfqLcPc^+7Uu#l(Bpxft{*4lv#*u7X9AOzDO z1D9?^jIo}?%iz(_dwLa{ex#T}76ZfN_Z-hwpus9y+4xaUu9cX}&P{XrZVWE{1^0yw zO;YhLEW!pJcbCt3L8~a7>jsaN{V3>tz6_7`&pi%GxZ=V3?3K^U+*ryLSb)8^IblJ0 zSRLNDvIxt)S}g30?s_3NX>F?NKIGrG_zB9@Z>uSW3k2es_H2kU;Rnn%j5qP)!XHKE zPB2mHP~tLCg4K_vH$xv`HbRsJwbZMUV(t=ez;Ec(vyHH)FbfLg`c61I$W_uBB>i^r z&{_P;369-&>23R%qNIULe=1~T$(DA`ev*EWZ6j(B$(te}x1WvmIll21zvygkS%vwG zzkR6Z#RKA2!z!C%M!O>!=Gr0(J0FP=-MN=5t-Ir)of50y10W}j`GtRCsXBakrKtG& zazmITDJMA0C51&BnLY)SY9r)NVTMs);1<=oosS9g31l{4ztjD3#+2H7u_|66b|_*O z;Qk6nalpqdHOjx|K&vUS_6ITgGll;TdaN*ta=M_YtyC)I9Tmr~VaPrH2qb6sd~=AcIxV+%z{E&0@y=DPArw zdV7z(G1hBx7hd{>(cr43^WF%4Y@PXZ?wPpj{OQ#tvc$pABJbvPGvdR`cAtHn)cSEV zrpu}1tJwQ3y!mSmH*uz*x0o|CS<^w%&KJzsj~DU0cLQUxk5B!hWE>aBkjJle8z~;s z-!A=($+}Jq_BTK5^B!`R>!MulZN)F=iXXeUd0w5lUsE5VP*H*oCy(;?S$p*TVvTxwAeWFB$jHyb0593)$zqalVlDX=GcCN1gU0 zlgU)I$LcXZ8Oyc2TZYTPu@-;7<4YYB-``Qa;IDcvydIA$%kHhJKV^m*-zxcvU4viy&Kr5GVM{IT>WRywKQ9;>SEiQD*NqplK-KK4YR`p0@JW)n_{TU3bt0 zim%;(m1=#v2}zTps=?fU5w^(*y)xT%1vtQH&}50ZF!9YxW=&7*W($2kgKyz1mUgfs zfV<*XVVIFnohW=|j+@Kfo!#liQR^x>2yQdrG;2o8WZR+XzU_nG=Ed2rK?ntA;K5B{ z>M8+*A4!Jm^Bg}aW?R?6;@QG@uQ8&oJ{hFixcfEnJ4QH?A4>P=q29oDGW;L;= z9-a0;g%c`C+Ai!UmK$NC*4#;Jp<1=TioL=t^YM)<<%u#hnnfSS`nq63QKGO1L8RzX z@MFDqs1z ztYmxDl@LU)5acvHk)~Z`RW7=aJ_nGD!mOSYD>5Odjn@TK#LY{jf?+piB5AM-CAoT_ z?S-*q7}wyLJzK>N%eMPuFgN)Q_otKP;aqy=D5f!7<=n(lNkYRXVpkB{TAYLYg{|(jtRqYmg$xH zjmq?B(RE4 zQx^~Pt}gxC2~l=K$$-sYy_r$CO(d=+b3H1MB*y_5g6WLaWTXn+TKQ|hNY^>Mp6k*$ zwkovomhu776vQATqT4blf~g;TY(MWCrf^^yfWJvSAB$p5l;jm@o#=!lqw+Lqfq>X= z$6~kxfm7`3q4zUEB;u4qa#BdJxO!;xGm)wwuisj{0y2x{R(IGMrsIzDY9LW>m!Y`= z04sx3IjnYvL<4JqxQ8f7qYd0s2Ig%`ytYPEMKI)s(LD}D@EY>x`VFtqvnADNBdeao zC96X+MxnwKmjpg{U&gP3HE}1=s!lv&D{6(g_lzyF3A`7Jn*&d_kL<;dAFx!UZ>hB8 z5A*%LsAn;VLp>3${0>M?PSQ)9s3}|h2e?TG4_F{}{Cs>#3Q*t$(CUc}M)I}8cPF6% z=+h(Kh^8)}gj(0}#e7O^FQ6`~fd1#8#!}LMuo3A0bN`o}PYsm!Y}sdOz$+Tegc=qT z8x`PH$7lvnhJp{kHWb22l;@7B7|4yL4UOOVM0MP_>P%S1Lnid)+k9{+3D+JFa#Pyf zhVc#&df87APl4W9X)F3pGS>@etfl=_E5tBcVoOfrD4hmVeTY-cj((pkn%n@EgN{0f zwb_^Rk0I#iZuHK!l*lN`ceJn(sI{$Fq6nN& zE<-=0_2WN}m+*ivmIOxB@#~Q-cZ>l136w{#TIJe478`KE7@=a{>SzPHsKLzYAyBQO zAtuuF$-JSDy_S@6GW0MOE~R)b;+0f%_NMrW(+V#c_d&U8Z9+ec4=HmOHw?gdjF(Lu zzra83M_BoO-1b3;9`%&DHfuUY)6YDV21P$C!Rc?mv&{lx#f8oc6?0?x zK08{WP65?#>(vPfA-c=MCY|%*1_<3D4NX zeVTi-JGl2uP_2@0F{G({pxQOXt_d{g_CV6b?jNpfUG9;8yle-^4KHRvZs-_2siata zt+d_T@U$&t*xaD22(fH(W1r$Mo?3dc%Tncm=C6{V9y{v&VT#^1L04vDrLM9qBoZ4@ z6DBN#m57hX7$C(=#$Y5$bJmwA$T8jKD8+6A!-IJwA{WOfs%s}yxUw^?MRZjF$n_KN z6`_bGXcmE#5e4Ym)aQJ)xg3Pg0@k`iGuHe?f(5LtuzSq=nS^5z>vqU0EuZ&75V%Z{ zYyhRLN^)$c6Ds{f7*FBpE;n5iglx5PkHfWrj3`x^j^t z7ntuV`g!9Xg#^3!x)l*}IW=(Tz3>Y5l4uGaB&lz{GDjm2D5S$CExLT`I1#n^lBH7Y zDgpMag@`iETKAI=p<5E#LTkwzVR@=yY|uBVI1HG|8h+d;G-qfuj}-ZR6fN>EfCCW z9~wRQoAPEa#aO?3h?x{YvV*d+NtPkf&4V0k4|L=uj!U{L+oLa(z#&iuhJr3-PjO3R z5s?=nn_5^*^Rawr>>Nr@K(jwkB#JK-=+HqwfdO<+P5byeim)wvqGlP-P|~Nse8=XF zz`?RYB|D6SwS}C+YQv+;}k6$-%D(@+t14BL@vM z2q%q?f6D-A5s$_WY3{^G0F131bbh|g!}#BKw=HQ7mx;Dzg4Z*bTLQSfo{ed{4}NZW zfrRm^Ca$rlE{Ue~uYv>R9{3smwATcdM_6+yWIO z*ZRH~uXE@#p$XTbCt5j7j2=86e{9>HIB6xDzV+vAo&B?KUiMP|ttOElepnl%|DPqL b{|{}U^kRn2wo}j7|0ATu<;8xA7zX}7|B6mN diff --git a/ballista/scheduler/ui/public/manifest.json b/ballista/scheduler/ui/public/manifest.json deleted file mode 100644 index 080d6c77a..000000000 --- a/ballista/scheduler/ui/public/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "short_name": "React App", - "name": "Create React App Sample", - "icons": [ - { - "src": "favicon.ico", - "sizes": "64x64 32x32 24x24 16x16", - "type": "image/x-icon" - }, - { - "src": "logo192.png", - "type": "image/png", - "sizes": "192x192" - }, - { - "src": "logo512.png", - "type": "image/png", - "sizes": "512x512" - } - ], - "start_url": ".", - "display": "standalone", - "theme_color": "#000000", - "background_color": "#ffffff" -} diff --git a/ballista/scheduler/ui/public/robots.txt b/ballista/scheduler/ui/public/robots.txt deleted file mode 100644 index dc045698d..000000000 --- a/ballista/scheduler/ui/public/robots.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# https://www.robotstxt.org/robotstxt.html -User-agent: * -Disallow: diff --git a/ballista/scheduler/ui/react-table-config.d.ts b/ballista/scheduler/ui/react-table-config.d.ts deleted file mode 100644 index 2c9994f91..000000000 --- a/ballista/scheduler/ui/react-table-config.d.ts +++ /dev/null @@ -1,146 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { - UseColumnOrderInstanceProps, - UseColumnOrderState, - UseExpandedHooks, - UseExpandedInstanceProps, - UseExpandedOptions, - UseExpandedRowProps, - UseExpandedState, - UseFiltersColumnOptions, - UseFiltersColumnProps, - UseFiltersInstanceProps, - UseFiltersOptions, - UseFiltersState, - UseGlobalFiltersColumnOptions, - UseGlobalFiltersInstanceProps, - UseGlobalFiltersOptions, - UseGlobalFiltersState, - UseGroupByCellProps, - UseGroupByColumnOptions, - UseGroupByColumnProps, - UseGroupByHooks, - UseGroupByInstanceProps, - UseGroupByOptions, - UseGroupByRowProps, - UseGroupByState, - UsePaginationInstanceProps, - UsePaginationOptions, - UsePaginationState, - UseResizeColumnsColumnOptions, - UseResizeColumnsColumnProps, - UseResizeColumnsOptions, - UseResizeColumnsState, - UseRowSelectHooks, - UseRowSelectInstanceProps, - UseRowSelectOptions, - UseRowSelectRowProps, - UseRowSelectState, - UseRowStateCellProps, - UseRowStateInstanceProps, - UseRowStateOptions, - UseRowStateRowProps, - UseRowStateState, - UseSortByColumnOptions, - UseSortByColumnProps, - UseSortByHooks, - UseSortByInstanceProps, - UseSortByOptions, - UseSortByState, -} from "react-table"; - -declare module "react-table" { - // take this file as-is, or comment out the sections that don't apply to your plugin configuration - - export interface TableOptions< - D extends Record - > extends UseExpandedOptions, - UseFiltersOptions, - UseGlobalFiltersOptions, - UseGroupByOptions, - UsePaginationOptions, - UseResizeColumnsOptions, - UseRowSelectOptions, - UseRowStateOptions, - UseSortByOptions, - // note that having Record here allows you to add anything to the options, this matches the spirit of the - // underlying js library, but might be cleaner if it's replaced by a more specific type that matches your - // feature set, this is a safe default. - Record {} - - export interface Hooks< - D extends Record = Record - > extends UseExpandedHooks, - UseGroupByHooks, - UseRowSelectHooks, - UseSortByHooks {} - - export interface TableInstance< - D extends Record = Record - > extends UseColumnOrderInstanceProps, - UseExpandedInstanceProps, - UseFiltersInstanceProps, - UseGlobalFiltersInstanceProps, - UseGroupByInstanceProps, - UsePaginationInstanceProps, - UseRowSelectInstanceProps, - UseRowStateInstanceProps, - UseSortByInstanceProps {} - - export interface TableState< - D extends Record = Record - > extends UseColumnOrderState, - UseExpandedState, - UseFiltersState, - UseGlobalFiltersState, - UseGroupByState, - UsePaginationState, - UseResizeColumnsState, - UseRowSelectState, - UseRowStateState, - UseSortByState {} - - export interface ColumnInterface< - D extends Record = Record - > extends UseFiltersColumnOptions, - UseGlobalFiltersColumnOptions, - UseGroupByColumnOptions, - UseResizeColumnsColumnOptions, - UseSortByColumnOptions {} - - export interface ColumnInstance< - D extends Record = Record - > extends UseFiltersColumnProps, - UseGroupByColumnProps, - UseResizeColumnsColumnProps, - UseSortByColumnProps {} - - export interface Cell< - D extends Record = Record, - V = any - > extends UseGroupByCellProps, - UseRowStateCellProps {} - - export interface Row< - D extends Record = Record - > extends UseExpandedRowProps, - UseGroupByRowProps, - UseRowSelectRowProps, - UseRowStateRowProps {} -} diff --git a/ballista/scheduler/ui/src/App.css b/ballista/scheduler/ui/src/App.css deleted file mode 100644 index bea95535e..000000000 --- a/ballista/scheduler/ui/src/App.css +++ /dev/null @@ -1,18 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ diff --git a/ballista/scheduler/ui/src/App.test.tsx b/ballista/scheduler/ui/src/App.test.tsx deleted file mode 100644 index 20dca216e..000000000 --- a/ballista/scheduler/ui/src/App.test.tsx +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import React from "react"; -import { render, screen } from "@testing-library/react"; -import App from "./App"; - -test("renders learn react link", () => { - render(); - const linkElement = screen.getByText(/learn react/i); - expect(linkElement).toBeInTheDocument(); -}); diff --git a/ballista/scheduler/ui/src/App.tsx b/ballista/scheduler/ui/src/App.tsx deleted file mode 100644 index c66fb37f7..000000000 --- a/ballista/scheduler/ui/src/App.tsx +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import React, { useState, useEffect } from "react"; -import { - Box, - Grid, - Tab, - TabList, - TabPanel, - TabPanels, - Tabs, - VStack, -} from "@chakra-ui/react"; -import { Header } from "./components/Header"; -import { Summary } from "./components/Summary"; -import { ExecutorsList } from "./components/ExecutorsList"; -import { QueriesList } from "./components/QueriesList"; -import { Footer } from "./components/Footer"; -import "./App.css"; - -const App: React.FunctionComponent = () => { - const [schedulerState, setSchedulerState] = useState(undefined); - const [jobs, setJobs] = useState(undefined); - const [executors, setExecutors] = useState(undefined); - - function getSchedulerState() { - return fetch(`/api/state`, { - method: "POST", - headers: { - Accept: "application/json", - }, - }) - .then((res) => res.json()) - .then((res) => setSchedulerState(res)); - } - - function getJobs() { - return fetch(`/api/jobs`, { - method: "POST", - headers: { - Accept: "application/json", - }, - }) - .then((res) => res.json()) - .then((res) => setJobs(res)); - } - - function getExecutors() { - return fetch(`/api/executors`, { - method: "POST", - headers: { - Accept: "application/json", - }, - }) - .then((res) => res.json()) - .then((res) => setExecutors(res)); - } - - useEffect(() => { - getSchedulerState(); - getJobs(); - getExecutors(); - }, []); - - return ( - - - -
- - - - Jobs - Executors - - - - - - - - - - -