diff --git a/charts/cluster/README.md b/charts/cluster/README.md index 8e8279e66..3f406cbc7 100644 --- a/charts/cluster/README.md +++ b/charts/cluster/README.md @@ -214,7 +214,27 @@ refer to the [CloudNativePG Documentation](https://cloudnative-pg.io/documentat | recovery.google.bucket | string | `""` | | | recovery.google.gkeEnvironment | bool | `false` | | | recovery.google.path | string | `"/"` | | -| recovery.method | string | `"backup"` | Available recovery methods: * `backup` - Recovers a CNPG cluster from a CNPG backup (PITR supported) Needs to be on the same cluster in the same namespace. * `object_store` - Recovers a CNPG cluster from a barman object store (PITR supported). * `pg_basebackup` - Recovers a CNPG cluster viaa streaming replication protocol. Useful if you want to migrate databases to CloudNativePG, even from outside Kubernetes. # TODO | +| recovery.import.databases | list | `[]` | Databases to import | +| recovery.import.postImportApplicationSQL | list | `[]` | List of SQL queries to be executed as a superuser in the application database right after is imported. To be used with extreme care. Only available in microservice type. | +| recovery.import.roles | list | `[]` | Roles to import | +| recovery.import.schemaOnly | bool | `false` | When set to true, only the pre-data and post-data sections of pg_restore are invoked, avoiding data import. | +| recovery.import.source.database | string | `""` | | +| recovery.import.source.host | string | `""` | | +| recovery.import.source.passwordSecret.create | bool | `false` | Whether to create a secret for the password | +| recovery.import.source.passwordSecret.key | string | `"password"` | The key in the secret containing the password | +| recovery.import.source.passwordSecret.name | string | `""` | Name of the secret containing the password | +| recovery.import.source.passwordSecret.value | string | `""` | The password value to use when creating the secret | +| recovery.import.source.port | int | `5432` | | +| recovery.import.source.sslCertSecret.key | string | `""` | | +| recovery.import.source.sslCertSecret.name | string | `""` | | +| recovery.import.source.sslKeySecret.key | string | `""` | | +| recovery.import.source.sslKeySecret.name | string | `""` | | +| recovery.import.source.sslMode | string | `"verify-full"` | | +| recovery.import.source.sslRootCertSecret.key | string | `""` | | +| recovery.import.source.sslRootCertSecret.name | string | `""` | | +| recovery.import.source.username | string | `""` | | +| recovery.import.type | string | `"microservice"` | One of `microservice` or `monolith.` See: https://cloudnative-pg.io/documentation/1.24/database_import/#how-it-works | +| recovery.method | string | `"backup"` | Available recovery methods: * `backup` - Recovers a CNPG cluster from a CNPG backup (PITR supported) Needs to be on the same cluster in the same namespace. * `object_store` - Recovers a CNPG cluster from a barman object store (PITR supported). * `pg_basebackup` - Recovers a CNPG cluster viaa streaming replication protocol. Useful if you want to migrate databases to CloudNativePG, even from outside Kubernetes. * `import` - Import one or more databases from an existing Postgres cluster. | | recovery.pgBaseBackup.database | string | `"app"` | Name of the database used by the application. Default: `app`. | | recovery.pgBaseBackup.owner | string | `""` | Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch | | recovery.pgBaseBackup.secret | string | `""` | Name of the owner of the database in the instance to be used by applications. Defaults to the value of the `database` key. | diff --git a/charts/cluster/templates/_bootstrap.tpl b/charts/cluster/templates/_bootstrap.tpl index 36f5b73cf..c81543cc4 100644 --- a/charts/cluster/templates/_bootstrap.tpl +++ b/charts/cluster/templates/_bootstrap.tpl @@ -3,7 +3,7 @@ bootstrap: initdb: {{- with .Values.cluster.initdb }} - {{- with (omit . "postInitApplicationSQL" "owner") }} + {{- with (omit . "postInitApplicationSQL" "owner" "import") }} {{- . | toYaml | nindent 4 }} {{- end }} {{- end }} @@ -43,33 +43,34 @@ bootstrap: {{- end }} externalClusters: -- name: pgBaseBackupSource - connectionParameters: - host: {{ .Values.recovery.pgBaseBackup.source.host | quote }} - port: {{ .Values.recovery.pgBaseBackup.source.port | quote }} - user: {{ .Values.recovery.pgBaseBackup.source.username | quote }} - dbname: {{ .Values.recovery.pgBaseBackup.source.database | quote }} - sslmode: {{ .Values.recovery.pgBaseBackup.source.sslMode | quote }} - {{- if .Values.recovery.pgBaseBackup.source.passwordSecret.name }} - password: - name: {{ default (printf "%s-pg-basebackup-password" (include "cluster.fullname" .)) .Values.recovery.pgBaseBackup.source.passwordSecret.name }} - key: {{ .Values.recovery.pgBaseBackup.source.passwordSecret.key }} - {{- end }} - {{- if .Values.recovery.pgBaseBackup.source.sslKeySecret.name }} - sslKey: - name: {{ .Values.recovery.pgBaseBackup.source.sslKeySecret.name }} - key: {{ .Values.recovery.pgBaseBackup.source.sslKeySecret.key }} - {{- end }} - {{- if .Values.recovery.pgBaseBackup.source.sslCertSecret.name }} - sslCert: - name: {{ .Values.recovery.pgBaseBackup.source.sslCertSecret.name }} - key: {{ .Values.recovery.pgBaseBackup.source.sslCertSecret.key }} - {{- end }} - {{- if .Values.recovery.pgBaseBackup.source.sslRootCertSecret.name }} - sslRootCert: - name: {{ .Values.recovery.pgBaseBackup.source.sslRootCertSecret.name }} - key: {{ .Values.recovery.pgBaseBackup.source.sslRootCertSecret.key }} - {{- end }} + {{- include "cluster.externalSourceCluster" (list "pgBaseBackupSource" .Values.recovery.pgBaseBackup.source) | nindent 2 }} + +{{- else if eq .Values.recovery.method "import" }} + initdb: + {{- with .Values.cluster.initdb }} + {{- with (omit . "owner" "import") }} + {{- . | toYaml | nindent 4 }} + {{- end }} + {{- end }} + {{- if .Values.cluster.initdb.owner }} + owner: {{ tpl .Values.cluster.initdb.owner . }} + {{- end }} + import: + source: + externalCluster: importSource + type: {{ .Values.recovery.import.type }} + databases: {{ .Values.recovery.import.databases | toJson }} + {{ with .Values.recovery.import.roles }} + roles: {{ . | toJson }} + {{- end }} + {{ with .Values.recovery.import.postImportApplicationSQL }} + postImportApplicationSQL: + {{- . | toYaml | nindent 6 }} + {{- end }} + schemaOnly: {{ .Values.recovery.import.schemaOnly }} + +externalClusters: + {{- include "cluster.externalSourceCluster" (list "importSource" .Values.recovery.import.source) | nindent 2 }} {{- else }} recovery: diff --git a/charts/cluster/templates/_external_source_cluster.tpl b/charts/cluster/templates/_external_source_cluster.tpl new file mode 100644 index 000000000..6d21e205e --- /dev/null +++ b/charts/cluster/templates/_external_source_cluster.tpl @@ -0,0 +1,33 @@ +{{- define "cluster.externalSourceCluster" -}} +{{- $name := first . -}} +{{- $config := last . -}} +- name: {{ first . }} + connectionParameters: + host: {{ $config.host | quote }} + port: {{ $config.port | quote }} + user: {{ $config.username | quote }} + {{- with $config.database }} + dbname: {{ . | quote }} + {{- end }} + sslmode: {{ $config.sslMode | quote }} + {{- if $config.passwordSecret.name }} + password: + name: {{ $config.passwordSecret.name }} + key: {{ $config.passwordSecret.key }} + {{- end }} + {{- if $config.sslKeySecret.name }} + sslKey: + name: {{ $config.sslKeySecret.name }} + key: {{ $config.sslKeySecret.key }} + {{- end }} + {{- if $config.sslCertSecret.name }} + sslCert: + name: {{ $config.sslCertSecret.name }} + key: {{ $config.sslCertSecret.key }} + {{- end }} + {{- if $config.sslRootCertSecret.name }} + sslRootCert: + name: {{ $config.sslRootCertSecret.name }} + key: {{ $config.sslRootCertSecret.key }} + {{- end }} +{{- end }} diff --git a/charts/cluster/test/postgresql-import/00-source-cluster-assert.yaml b/charts/cluster/test/postgresql-import/00-source-cluster-assert.yaml new file mode 100644 index 000000000..90ea90fd5 --- /dev/null +++ b/charts/cluster/test/postgresql-import/00-source-cluster-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: source-cluster +status: + readyInstances: 1 diff --git a/charts/cluster/test/postgresql-import/00-source-cluster.yaml b/charts/cluster/test/postgresql-import/00-source-cluster.yaml new file mode 100644 index 000000000..790e20836 --- /dev/null +++ b/charts/cluster/test/postgresql-import/00-source-cluster.yaml @@ -0,0 +1,9 @@ +type: postgresql +mode: "standalone" +cluster: + instances: 1 + superuserSecret: source-cluster-superuser + storage: + size: 256Mi +backups: + enabled: false diff --git a/charts/cluster/test/postgresql-import/00-source-superuser-password.yaml b/charts/cluster/test/postgresql-import/00-source-superuser-password.yaml new file mode 100644 index 000000000..b88a8084d --- /dev/null +++ b/charts/cluster/test/postgresql-import/00-source-superuser-password.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: source-cluster-superuser +type: Opaque +data: + username: "cG9zdGdyZXM=" + password: "cG9zdGdyZXM=" diff --git a/charts/cluster/test/postgresql-import/01-data_write-assert.yaml b/charts/cluster/test/postgresql-import/01-data_write-assert.yaml new file mode 100644 index 000000000..831f963d9 --- /dev/null +++ b/charts/cluster/test/postgresql-import/01-data_write-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: data-write +status: + succeeded: 1 diff --git a/charts/cluster/test/postgresql-import/01-data_write.yaml b/charts/cluster/test/postgresql-import/01-data_write.yaml new file mode 100644 index 000000000..d80a12770 --- /dev/null +++ b/charts/cluster/test/postgresql-import/01-data_write.yaml @@ -0,0 +1,31 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: data-write +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: data-write + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: source-cluster-superuser + key: username + - name: DB_PASS + valueFrom: + secretKeyRef: + name: source-cluster-superuser + key: password + - name: DB_URI + value: postgres://$(DB_USER):$(DB_PASS)@source-cluster-rw:5432 + image: alpine:3.19 + command: ['sh', '-c'] + args: + - | + apk --no-cache add postgresql-client + psql "$DB_URI" -c "CREATE DATABASE mygooddb;" + psql "$DB_URI/mygooddb" -c "CREATE TABLE mygoodtable (id serial PRIMARY KEY);" + psql "$DB_URI/mygooddb" -c "INSERT INTO mygoodtable VALUES (314159265);" diff --git a/charts/cluster/test/postgresql-import/02-import-cluster-assert.yaml b/charts/cluster/test/postgresql-import/02-import-cluster-assert.yaml new file mode 100644 index 000000000..a194edc57 --- /dev/null +++ b/charts/cluster/test/postgresql-import/02-import-cluster-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: import-cluster +status: + readyInstances: 2 diff --git a/charts/cluster/test/postgresql-import/02-import-cluster.yaml b/charts/cluster/test/postgresql-import/02-import-cluster.yaml new file mode 100644 index 000000000..bf3e86d39 --- /dev/null +++ b/charts/cluster/test/postgresql-import/02-import-cluster.yaml @@ -0,0 +1,30 @@ +type: postgresql +mode: "recovery" +recovery: + method: "import" + import: + type: "microservice" + databases: [ "mygooddb" ] + source: + host: "source-cluster-rw" + username: "postgres" + passwordSecret: + name: source-cluster-superuser + key: password + sslMode: "require" + sslKeySecret: + name: source-cluster-replication + key: tls.key + sslCertSecret: + name: source-cluster-replication + key: tls.crt + +cluster: + instances: 2 + storage: + size: 256Mi + initdb: + database: mygooddb + +backups: + enabled: false diff --git a/charts/cluster/test/postgresql-import/03-data_test-assert.yaml b/charts/cluster/test/postgresql-import/03-data_test-assert.yaml new file mode 100644 index 000000000..04df941e4 --- /dev/null +++ b/charts/cluster/test/postgresql-import/03-data_test-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: data-test +status: + succeeded: 1 diff --git a/charts/cluster/test/postgresql-import/03-data_test.yaml b/charts/cluster/test/postgresql-import/03-data_test.yaml new file mode 100644 index 000000000..7df77cb14 --- /dev/null +++ b/charts/cluster/test/postgresql-import/03-data_test.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: data-test +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: data-test + env: + - name: DB_URI + valueFrom: + secretKeyRef: + name: import-cluster-superuser + key: uri + image: alpine:3.19 + command: ['sh', '-c'] + args: + - | + apk --no-cache add postgresql-client + DB_URI=$(echo $DB_URI | sed "s|/\*|/|" ) + test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $$mygoodtable$$)' --csv -q 2>/dev/null)" = "t" + test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT EXISTS (SELECT FROM mygoodtable WHERE id = 314159265)' --csv -q 2>/dev/null)" = "t" diff --git a/charts/cluster/test/postgresql-import/04-import-cluster-schema_only-assert.yaml b/charts/cluster/test/postgresql-import/04-import-cluster-schema_only-assert.yaml new file mode 100644 index 000000000..f461b2bc1 --- /dev/null +++ b/charts/cluster/test/postgresql-import/04-import-cluster-schema_only-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: import-schemaonly-cluster +status: + readyInstances: 2 diff --git a/charts/cluster/test/postgresql-import/04-import-cluster-schema_only.yaml b/charts/cluster/test/postgresql-import/04-import-cluster-schema_only.yaml new file mode 100644 index 000000000..7cae8a3b7 --- /dev/null +++ b/charts/cluster/test/postgresql-import/04-import-cluster-schema_only.yaml @@ -0,0 +1,31 @@ +type: postgresql +mode: "recovery" +recovery: + method: "import" + import: + type: "microservice" + databases: [ "mygooddb" ] + schemaOnly: true + source: + host: "source-cluster-rw" + username: "postgres" + passwordSecret: + name: source-cluster-superuser + key: password + sslMode: "require" + sslKeySecret: + name: source-cluster-replication + key: tls.key + sslCertSecret: + name: source-cluster-replication + key: tls.crt + +cluster: + instances: 2 + storage: + size: 256Mi + initdb: + database: mygooddb + +backups: + enabled: false diff --git a/charts/cluster/test/postgresql-import/05-data_test-assert.yaml b/charts/cluster/test/postgresql-import/05-data_test-assert.yaml new file mode 100644 index 000000000..b29534fcc --- /dev/null +++ b/charts/cluster/test/postgresql-import/05-data_test-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: data-test-schemaonly +status: + succeeded: 1 diff --git a/charts/cluster/test/postgresql-import/05-data_test.yaml b/charts/cluster/test/postgresql-import/05-data_test.yaml new file mode 100644 index 000000000..05ed21700 --- /dev/null +++ b/charts/cluster/test/postgresql-import/05-data_test.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: data-test-schemaonly +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: data-test + env: + - name: DB_URI + valueFrom: + secretKeyRef: + name: import-schemaonly-cluster-superuser + key: uri + image: alpine:3.19 + command: ['sh', '-c'] + args: + - | + apk --no-cache add postgresql-client + DB_URI=$(echo $DB_URI | sed "s|/\*|/|" ) + test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $$mygoodtable$$)' --csv -q 2>/dev/null)" = "t" + test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT EXISTS (SELECT FROM mygoodtable WHERE id = 314159265)' --csv -q 2>/dev/null)" = "f" diff --git a/charts/cluster/test/postgresql-import/chainsaw-test.yaml b/charts/cluster/test/postgresql-import/chainsaw-test.yaml new file mode 100644 index 000000000..f4b431f6b --- /dev/null +++ b/charts/cluster/test/postgresql-import/chainsaw-test.yaml @@ -0,0 +1,97 @@ +## +# This test the CNPG PostgreSQL import capability. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: postgresql-import +spec: + timeouts: + apply: 1s + assert: 2m + cleanup: 1m + steps: + - name: Install the external PostgreSQL cluster + try: + - apply: + file: ./00-source-superuser-password.yaml + - script: + content: | + helm upgrade \ + --install \ + --namespace $NAMESPACE \ + --values ./00-source-cluster.yaml \ + --wait \ + source ../../ + - assert: + file: ./00-source-cluster-assert.yaml + - apply: + file: ./01-data_write.yaml + - assert: + file: ./01-data_write-assert.yaml + - name: Install the import cluster + timeouts: + assert: 5m + try: + - script: + content: | + helm upgrade \ + --install \ + --namespace $NAMESPACE \ + --values ./02-import-cluster.yaml \ + --wait \ + import ../../ + - assert: + file: ./02-import-cluster-assert.yaml + catch: + - describe: + apiVersion: postgresql.cnpg.io/v1 + kind: Cluster + - name: Verify the data exists + try: + - apply: + file: ./03-data_test.yaml + - assert: + file: ./03-data_test-assert.yaml + catch: + - describe: + apiVersion: batch/v1 + kind: Job + - podLogs: + selector: batch.kubernetes.io/job-name=data-test + - name: Install the schema-only import cluster + timeouts: + assert: 5m + try: + - script: + content: | + helm upgrade \ + --install \ + --namespace $NAMESPACE \ + --values ./04-import-cluster-schema_only.yaml \ + --wait \ + import-schemaonly ../../ + - assert: + file: ./04-import-cluster-schema_only-assert.yaml + catch: + - describe: + apiVersion: postgresql.cnpg.io/v1 + kind: Cluster + - name: Verify only the schema exists + try: + - apply: + file: ./05-data_test.yaml + - assert: + file: ./05-data_test-assert.yaml + catch: + - describe: + apiVersion: batch/v1 + kind: Job + - podLogs: + selector: batch.kubernetes.io/job-name=data-test-schemaonly + - name: Cleanup + try: + - script: + content: | + helm uninstall --namespace $NAMESPACE source + helm uninstall --namespace $NAMESPACE import + helm uninstall --namespace $NAMESPACE import-schemaonly diff --git a/charts/cluster/test/postgresql-pg_basebackup/00-source-cluster.yaml b/charts/cluster/test/postgresql-pg_basebackup/00-source-cluster.yaml index c11fed595..5d0baa1c0 100644 --- a/charts/cluster/test/postgresql-pg_basebackup/00-source-cluster.yaml +++ b/charts/cluster/test/postgresql-pg_basebackup/00-source-cluster.yaml @@ -2,5 +2,7 @@ type: postgresql mode: "standalone" cluster: instances: 1 + storage: + size: 256Mi backups: - enabled: false \ No newline at end of file + enabled: false diff --git a/charts/cluster/test/postgresql-pg_basebackup/02-pg_basebackup-cluster.yaml b/charts/cluster/test/postgresql-pg_basebackup/02-pg_basebackup-cluster.yaml index d389200e8..310074e1d 100644 --- a/charts/cluster/test/postgresql-pg_basebackup/02-pg_basebackup-cluster.yaml +++ b/charts/cluster/test/postgresql-pg_basebackup/02-pg_basebackup-cluster.yaml @@ -17,6 +17,8 @@ recovery: cluster: instances: 2 + storage: + size: 256Mi backups: - enabled: false \ No newline at end of file + enabled: false diff --git a/charts/cluster/values.schema.json b/charts/cluster/values.schema.json index cb181a737..4194914dd 100644 --- a/charts/cluster/values.schema.json +++ b/charts/cluster/values.schema.json @@ -431,6 +431,96 @@ } } }, + "import": { + "type": "object", + "properties": { + "databases": { + "type": "array" + }, + "postImportApplicationSQL": { + "type": "array" + }, + "roles": { + "type": "array" + }, + "schemaOnly": { + "type": "boolean" + }, + "source": { + "type": "object", + "properties": { + "database": { + "type": "string" + }, + "host": { + "type": "string" + }, + "passwordSecret": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "port": { + "type": "integer" + }, + "sslCertSecret": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "sslKeySecret": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "sslMode": { + "type": "string" + }, + "sslRootCertSecret": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "username": { + "type": "string" + } + } + }, + "type": { + "type": "string" + } + } + }, "method": { "type": "string" }, diff --git a/charts/cluster/values.yaml b/charts/cluster/values.yaml index 1d650806b..e2a1a3f3e 100644 --- a/charts/cluster/values.yaml +++ b/charts/cluster/values.yaml @@ -33,7 +33,8 @@ recovery: # * `backup` - Recovers a CNPG cluster from a CNPG backup (PITR supported) Needs to be on the same cluster in the same namespace. # * `object_store` - Recovers a CNPG cluster from a barman object store (PITR supported). # * `pg_basebackup` - Recovers a CNPG cluster viaa streaming replication protocol. Useful if you want to - # migrate databases to CloudNativePG, even from outside Kubernetes. # TODO + # migrate databases to CloudNativePG, even from outside Kubernetes. + # * `import` - Import one or more databases from an existing Postgres cluster. method: backup ## -- Point in time recovery target. Specify one of the following: @@ -125,6 +126,45 @@ recovery: name: "" key: "" + # See: https://cloudnative-pg.io/documentation/1.24/cloudnative-pg.v1/#postgresql-cnpg-io-v1-Import + import: + # -- One of `microservice` or `monolith.` + # See: https://cloudnative-pg.io/documentation/1.24/database_import/#how-it-works + type: "microservice" + # -- Databases to import + databases: [] + # -- Roles to import + roles: [] + # -- List of SQL queries to be executed as a superuser in the application database right after is imported. + # To be used with extreme care. Only available in microservice type. + postImportApplicationSQL: [] + # -- When set to true, only the pre-data and post-data sections of pg_restore are invoked, avoiding data import. + schemaOnly: false + source: + host: "" + port: 5432 + username: "" + database: "" + sslMode: "verify-full" + passwordSecret: + # -- Whether to create a secret for the password + create: false + # -- Name of the secret containing the password + name: "" + # -- The key in the secret containing the password + key: "password" + # -- The password value to use when creating the secret + value: "" + sslKeySecret: + name: "" + key: "" + sslCertSecret: + name: "" + key: "" + sslRootCertSecret: + name: "" + key: "" + cluster: # -- Number of instances