Appearance
operator yaml
yaml
# The main role for the operator
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: mysql-operator
labels:
app.kubernetes.io/name: mysql-operator
app.kubernetes.io/instance: mysql-operator
app.kubernetes.io/component: controller
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "patch"]
- apiGroups: [""]
resources: ["pods/status"]
verbs: ["get", "patch", "update", "watch"]
# Kopf needs patch on secrets or the sidecar will throw
# The operator needs this verb to be able to pass it to the sidecar
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "create", "list", "watch", "patch"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "create", "update", "list", "watch", "patch", "delete"]
- apiGroups: [""]
resources: ["services"]
verbs: ["get", "create", "list", "update", "delete", "patch"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get", "create", "patch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "patch", "update"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings"]
verbs: ["get", "create", "patch", "delete"]
- apiGroups: ["policy"]
resources: ["poddisruptionbudgets"]
verbs: ["get", "create"]
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["create"]
- apiGroups: ["batch"]
resources: ["cronjobs"]
verbs: ["get", "create", "update", "delete"]
- apiGroups: ["apps"]
resources: ["deployments", "statefulsets"]
verbs: ["get", "list", "create", "patch", "update", "watch", "delete"]
- apiGroups: ["mysql.oracle.com"]
resources: ["*"]
verbs: ["*"]
- apiGroups: ["zalando.org"]
resources: ["*"]
verbs: ["get", "patch", "list", "watch"]
# Kopf: runtime observation of namespaces & CRDs (addition/deletion).
- apiGroups: [apiextensions.k8s.io]
resources: [customresourcedefinitions]
verbs: [list, watch]
- apiGroups: [""]
resources: [namespaces]
verbs: [list, watch]
- apiGroups: ["monitoring.coreos.com"]
resources: ["servicemonitors"]
verbs: ["get", "create", "patch", "update", "delete"]
---
# role for the server sidecar
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: mysql-sidecar
labels:
app.kubernetes.io/name: mysql-operator
app.kubernetes.io/instance: mysql-operator
app.kubernetes.io/component: controller
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "patch"]
- apiGroups: [""]
resources: ["pods/status"]
verbs: ["get", "patch", "update", "watch"]
# Kopf needs patch on secrets or the sidecar will throw
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "create", "list", "watch", "patch"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "create", "list", "watch", "patch"]
- apiGroups: [""]
resources: ["services"]
verbs: ["get", "create", "list", "update"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get", "create"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "patch", "update"]
- apiGroups: ["apps"]
resources: ["deployments"]
verbs: ["get", "patch"]
- apiGroups: ["mysql.oracle.com"]
resources: ["innodbclusters"]
verbs: ["get", "watch", "list", "patch"]
- apiGroups: ["mysql.oracle.com"]
resources: ["mysqlbackups"]
verbs: ["create", "get", "list", "patch", "update", "watch", "delete"]
- apiGroups: ["mysql.oracle.com"]
resources: ["mysqlbackups/status"]
verbs: ["get", "patch", "update", "watch"]
---
# Role for the switchover CronJob
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: mysql-switchover
labels:
app.kubernetes.io/name: mysql-operator
app.kubernetes.io/instance: mysql-operator
app.kubernetes.io/component: controller
rules:
# Kopf needs patch on secrets or the sidecar will throw
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "patch", "update"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
- apiGroups: ["mysql.oracle.com"]
resources: ["innodbclusters"]
verbs: ["get"]
- apiGroups: ["mysql.oracle.com"]
resources: ["innodbclusters/status"]
verbs: ["get", "patch"]
---
# Give access to the operator
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: mysql-operator-rolebinding
labels:
app.kubernetes.io/name: mysql-operator
app.kubernetes.io/instance: mysql-operator
app.kubernetes.io/component: controller
subjects:
- kind: ServiceAccount
name: mysql-operator-sa
namespace: mysql-operator
# TODO The following entry is for dev purposes only
#- kind: Group
# name: system:serviceaccounts
# apiGroup: rbac.authorization.k8s.io
roleRef:
kind: ClusterRole
name: mysql-operator
apiGroup: rbac.authorization.k8s.io
---
apiVersion: zalando.org/v1
kind: ClusterKopfPeering
metadata:
#Should be the same as spec.template.spec.containers[0].env['OPERATOR_PEERING_NAME']
name: mysql-operator
labels:
app.kubernetes.io/name: mysql-operator
app.kubernetes.io/instance: mysql-operator
app.kubernetes.io/component: controller
---
apiVersion: v1
kind: Namespace
metadata:
name: mysql-operator
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: mysql-operator-sa
namespace: mysql-operator
labels:
app.kubernetes.io/name: mysql-operator
app.kubernetes.io/instance: mysql-operator
app.kubernetes.io/component: controller
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: mysql-operator
namespace: mysql-operator
labels:
version: "1.0"
app.kubernetes.io/name: mysql-operator
app.kubernetes.io/instance: mysql-operator
app.kubernetes.io/version: 26.7.0-2.3.0
app.kubernetes.io/component: controller
app.kubernetes.io/managed-by: mysql-operator
app.kubernetes.io/created-by: mysql-operator
spec:
replicas: 1
selector:
matchLabels:
name: mysql-operator
template:
metadata:
labels:
name: mysql-operator
app.kubernetes.io/name: mysql-operator
app.kubernetes.io/instance: mysql-operator
app.kubernetes.io/component: controller
spec:
securityContext:
runAsNonRoot: true
containers:
- name: mysql-operator
image: 10.1.250.75/k8s/mysql/community-operator:26.7.0-2.3.0
imagePullPolicy: IfNotPresent
args:
[
"mysqlsh",
"--log-level=@INFO",
"--pym",
"mysqloperator",
"operator",
]
env:
- name: MYSQLSH_USER_CONFIG_HOME
value: /mysqlsh
- name: MYSQLSH_CREDENTIAL_STORE_SAVE_PASSWORDS
value: never
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: OPERATOR_DEPLOYMENT_NAME
value: "mysql-operator"
# used when when OPERATOR_STANDALONE is false
- name: OPERATOR_PEERING_NAME
value: "mysql-operator"
- name: OPERATOR_ROLE_NAME
value: "mysql-operator"
- name: SIDECAR_ROLE_NAME
value: "mysql-sidecar"
- name: SWITCHOVER_ROLE_NAME
value: "mysql-switchover"
# These envs define operator topology for raw-manifest installs and upgrades.
# Standalone also requires a safe Deployment strategy.
# Comma-separated explicit watched namespaces. Empty means cluster-wide.
# Standalone and non-standalone use the same explicit namespace grammar.
- name: OPERATOR_NAMESPACES
value: ""
- name: OPERATOR_STANDALONE
value: "false"
- name: MYSQL_OPERATOR_DEBUG
value: "0"
readinessProbe:
exec:
command:
- cat
- /tmp/mysql-operator-ready
initialDelaySeconds: 1
periodSeconds: 3
volumeMounts:
- name: mysqlsh-home
mountPath: /mysqlsh
- name: tmpdir
mountPath: /tmp
securityContext:
capabilities:
drop:
- ALL
runAsUser: 2
allowPrivilegeEscalation: false
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
volumes:
- name: mysqlsh-home
emptyDir: {}
- name: tmpdir
emptyDir: {}
serviceAccountName: mysql-operator-sacrds yaml
yaml
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: innodbclusters.mysql.oracle.com
spec:
group: mysql.oracle.com
versions:
- name: v2
served: true
storage: true
schema:
openAPIV3Schema:
type: object
required: ["spec"]
properties:
metadata:
type: object
properties:
name:
type: string
maxLength: 40
spec:
type: object
required: ["secretName"]
properties:
secretName:
type: string
description: "Name of a generic type Secret containing root/default account password"
tlsCASecretName:
type: string
description: "Name of a generic type Secret containing CA (ca.pem) and optional CRL (crl.pem) for SSL"
tlsSecretName:
type: string
description: "Name of a TLS type Secret containing Server certificate and private key for SSL"
tlsUseSelfSigned:
type: boolean
default: false
description: "Enables use of self-signed TLS certificates, reducing or disabling TLS based security verifications"
meb:
type: object
properties:
tlsSecretName:
type: string
description: "Name of a Secret containing MySQL Enterprise Backup client certificate and private key"
version:
type: string
pattern: '^\d+\.\d+\.\d+(-.+)?'
description: "MySQL Server version"
edition:
type: string
pattern: "^(community|enterprise)$"
description: "MySQL Server Edition (community or enterprise)"
imageRepository:
type: string
description: "Repository where images are pulled from; defaults to container-registry.oracle.com/mysql"
imagePullPolicy:
type: string
description: "Defaults to Always, but set to IfNotPresent in deploy-operator.yaml when deploying Operator"
imagePullSecrets:
type: array
items:
type: object
properties:
name:
type: string
serviceAccountName:
type: string
baseServerId:
type: integer
minimum: 0
maximum: 4294967195
default: 1000
description: "Base value for MySQL server_id for instances in the cluster"
datadirVolumeClaimTemplate:
type: object
x-kubernetes-preserve-unknown-fields: true
description: "Template for a PersistentVolumeClaim, to be used as datadir"
datadirPermissions:
type: object
properties:
setRightsUsingInitContainer:
type: boolean
default: true
description: "Whether to use an init container to set at start the DataDir permissions "
fsGroupChangePolicy:
type: string
description: "Optional fsGroupChangePolicy value to be set in the pod security context. Some possible values are OnRootMismatch and Always. For more information check the official Kubernetes documentation"
default: ""
mycnf:
type: string
description: "Custom configuration additions for my.cnf"
instances:
type: integer
minimum: 1
maximum: 9
default: 1
description: "Number of MySQL replica instances for the cluster"
podSpec:
type: object
x-kubernetes-preserve-unknown-fields: true
description: "Customization specification to be merged into the InnoDBCluster StatefulSet pod template. IMPORTANT: podSpec is regarded only during StatefulSet buildup and the operator neither checks the values in podSpec nor later watches for changes of this field. Thus if changes are made to this field the InnoDBCluster StatefulSet will not be updated"
podAnnotations:
type: object
x-kubernetes-preserve-unknown-fields: true
podLabels:
type: object
x-kubernetes-preserve-unknown-fields: true
keyring:
type: object
description: "Keyring specification"
properties:
kmip:
type: object
description: "Keyring 'KMIP' specification"
required: ["configuration", "server"]
properties:
configuration:
type: string
default: ""
description: "Name of a secret that contains TLS certificates"
cacheKeys:
type: boolean
default: true
description: "Whether the keys are cached by the MySQL Server in RAM in plaintext. If set to false the keys are decrypted on every access"
server:
type: string
default: ""
description: "Primary OKV Server host with port number in the format <host>:<port>"
standbyServer:
type: array
default: []
description: "A list of standby servers in the format <host>:<port>"
items:
type: string
file:
type: object
description: "Keyring 'File' specification"
required: ["storage"]
properties:
fileName:
type: string
default: "mysql_keyring"
description: "Path to the keyring file name inside the storage volume (will be prefixed by mount path)"
readOnly:
type: boolean
default: false
description: "Whether to open the keyring file in read-only mode"
storage:
type: object
description : "Specification of the volume to be mounted where the keyring file resides"
x-kubernetes-preserve-unknown-fields: true
encryptedFile:
type: object
description: "Keyring 'Encrypted File' specification"
required: ["storage", "password"]
properties:
fileName:
type: string
default: "mysql_keyring"
description: "Path to the keyring file name inside the storage volume (will be prefixed by mount path)"
readOnly:
type: boolean
default: false
description: "Whether to open the keyring file in read-only mode"
password:
type: string
description: "Name of a secret that contains password for the keyring in the key 'keyring_password'"
storage:
type: object
description : "Specification of the volume to be mounted where the keyring file resides"
x-kubernetes-preserve-unknown-fields: true
hashicorp:
type: object
description: "Keyring 'HashiCorp Vault' specification"
required: ['serverUrl', 'auth', 'storePath' ]
properties:
caCertificate:
type: string
description: "Name of a ConfigMap with a ca.pem field with CA certificate bundle file with the CA of the HashiCorp Vault"
caching:
type: boolean
default: false
description: "Whether to enable in-memory key cache"
serverUrl:
type: string
description: "URL of HashiCorp Vault server"
storePath:
type: string
description: "Path within HashiCorp Vault server which is writeable"
auth:
type: object
oneOf:
- required: ['approle']
- required: ['token']
description: "Authentication settings, either 'approle' or 'token' must be set"
properties:
approle:
type: object
description: 'AppRole authentication with role and secret'
required: ['authSecret']
properties:
authenticationPath:
type: string
description: "Authentication path where AppRole authentication is enabled within the HashiCorp Vault server"
default: "/v1/auth/approle/login"
authSecret:
type: string
description: "Name of a Secret in this namespace, with fields 'role_id.txt' and 'secret_id.txt' containing credentials"
token:
type: object
description: 'Token authentication'
required: ['tokenSecret']
properties:
tokenSecret:
type: string
description: "Name of a Secret in this namespace, with a filed 'token.txt' containing authentication token"
oci:
type: object
description: "Keyring 'OCI' specification"
required: ["user", "keySecret", "keyFingerprint", "tenancy"]
properties:
user:
type: string
description: "User identifier in the form of ocid1.user.oc1..."
pattern: '^ocid1\.user\.'
keySecret:
type: string
description: "A secret that contains the private key under the field 'privatekey'"
keyFingerprint:
type: string
description: "Private key fingerprint"
pattern: '([0-9a-f]{2}:){15}[0-9a-f]{2}$'
tenancy:
type: string
description: "Tenancy identifier in the form ocid1.tenancy.oc1..."
pattern: '^ocid1\.tenancy\.'
compartment:
type: string
description: "Compartment identifier in the form ocid1.compartment.oc1..."
pattern: '^ocid1\.compartment\.'
virtualVault:
type: string
description: "Vault identifier in the form ocid1.vault.oc1..."
pattern: '^ocid1\.vault\.'
masterKey:
type: string
description: "Master key identified in the form ocid1.key.oc1..."
pattern: '^ocid1\.key\.'
endpoints:
type: object
description: ""
properties:
encryption:
type: string
description: "Encryption endpoint URI like {identifier}-crypto.kms.{region}.oraclecloud.com"
management:
type: string
description: "Management endpoint URI like {identifier}-management.kms.{region}.oraclecloud.com"
vaults:
type: string
description: "Vaults endpoint URI like vaults.{region}.oci.oraclecloud.com"
secrets:
type: string
description: "Secrets endpoint URI like secrets.vaults.{region}.oci.oraclecloud.com"
caCertificate:
type: string
description: "Secret that contains ca.crt field with CA certificate bundle file that the keyring_oci plugin uses for Oracle Cloud Infrastructure certificate verification"
initDB:
type: object
properties:
clusterSet:
type: object
required: ["targetUrl", "secretKeyRef"]
properties:
targetUrl:
type: string
description: "URL of a cluster, which is a ClusterSet to join to"
rootUser:
type: string
default: "root"
description: "User name used for setup"
secretKeyRef:
type: object
required: ["name"]
properties:
name:
type: string
description: "Secret name with key 'rootPassword' storing the password for the user specified in rootUser"
clone:
type: object
required: ["donorUrl", "secretKeyRef"]
properties:
donorUrl:
type: string
description: "URL of the cluster to clone from"
rootUser:
type: string
default: "root"
description: "User name used for cloning"
secretKeyRef:
type: object
required: ["name"]
properties:
name:
type: string
description: "Secret name with key 'rootPassword' storing the password for the user specified in rootUser"
dump:
type: object
required: ["storage"]
properties:
name:
type: string
description: "Name of the dump. Not used by the operator, but a descriptive hint for the cluster administrator"
path:
type: string
description: "Path to the dump in the PVC. Use when specifying persistentVolumeClaim. Omit for ociObjectStorage, S3, or azure."
options:
type: object
description: "A dictionary of key-value pairs passed directly to MySQL Shell's loadDump()"
x-kubernetes-preserve-unknown-fields: true
storage:
type: object
properties:
ociObjectStorage:
type: object
required: ["bucketName", "prefix", "credentials"]
properties:
bucketName:
type: string
description: "Name of the OCI bucket where the dump is stored"
prefix:
type: string
description: "Path in the bucket where the dump files are stored"
credentials:
type: string
description: "Name of a Secret with data for accessing the bucket"
s3:
type: object
required: ["bucketName", "prefix", "config"]
properties:
bucketName:
type: string
description: "Name of the S3 bucket where the dump is stored"
prefix:
type: string
description: "Path in the bucket where the dump files are stored"
config:
type: string
description: "Name of a Secret with S3 configuration and credentials"
profile:
type: string
default: ""
description: "Profile being used in configuration files"
endpoint:
type: string
description: "Override endpoint URL"
azure:
type: object
required: ["containerName", "prefix", "config"]
properties:
containerName:
type: string
description: "Name of the Azure BLOB Storage container where the dump is stored"
prefix:
type: string
description: "Path in the container where the dump files are stored"
config:
type: string
description: "Name of a Secret with Azure BLOB Storage configuration and credentials"
persistentVolumeClaim:
type: object
description : "Specification of the PVC to be used. Used 'as is' in the cloning pod."
x-kubernetes-preserve-unknown-fields: true
x-kubernetes-preserve-unknown-fields: true
meb:
type: object
required: ["storage", "fullBackup"]
properties:
storage:
type: object
#oneOf:
#- required: ["ociObjectStorage", "s3"]
properties:
ociObjectStorage:
type: object
description: "Restore from OCI ObjectStorage using an PAR"
required: ["credentials"]
properties:
credentials:
type: string
description: "Secret Containing PAR Base URL under key parBaseUrl"
s3:
description: "S3 style storage"
type: object
required: ['region', 'bucket', 'objectKeyPrefix', 'credentials']
properties:
region:
type: string
description: "S3 Region"
bucket:
type: string
description: "S3 Bucket Name"
objectKeyPrefix:
type: string
description: "S3 Object Key Prefix, this will be extended with the name of the actual backup"
credentials:
type: string
description: "Name of a Secret holding iaccessKeyId and secretAccessKey properties"
host:
type: string
description: "Override hostname for S3-compatible backends"
fullBackup:
type: string
description: "Name of the full backup to restore"
incrementalBackups:
type: array
description: "Incremental backups to apply on top of the fullBackup"
items:
type: string
pitr:
type: object
description: "Point-in-time recovery using a backup file on top of fullBackup and optional incrementalBackups"
required: ["backupFile"]
properties:
backupFile:
type: string
description: "Backup containing binary log to apply"
binlogName:
type: string
description: "Basename of binlogs to use for PITR. Often this is the server_name of the backup source"
end:
type: object
description: "End condition for not applying the full log"
#oneOf: ['afterGtids', 'beforeGtids']
properties:
afterGtids:
type: string
description: "GTID set describing last transactions to apply, see SQL_AFTER_GTIDS in CHANGE REPLICATION SOURCE statement documentation"
beforeGtids:
type: string
description: "GTID set describing last transactions to apply, see SQL_BEFORE_GTID in CHANGE REPLICATION SOURCE statement documentation"
extraOptions:
description: "Additonal Command Line Options being passed to MySQL Enterprise Backup"
type: array
items:
type: string
x-kubernetes-preserve-unknown-fields: true
router:
type: object
description: "MySQL Router specification"
properties:
instances:
type: integer
minimum: 0
default: 1
description: "Number of MySQL Router instances to deploy"
tlsSecretName:
type: string
description: "Name of a TLS type Secret containing MySQL Router certificate and private key used for SSL"
version:
type: string
pattern: '^\d+\.\d+\.\d+(-.+)?'
description: "Override MySQL Router version"
podSpec:
type: object
x-kubernetes-preserve-unknown-fields: true
description: "Customization specification to be merged into the Router Deployment pod template. IMPORTANT: podSpec is regarded only during Deployment buildup and the operator neither checks the values in podSpec nor later watches for changes of this field. Thus if changes are made to this field the Router Deployment will not be updated"
podAnnotations:
type: object
x-kubernetes-preserve-unknown-fields: true
podLabels:
type: object
x-kubernetes-preserve-unknown-fields: true
bootstrapOptions:
description: "Command line options passed to MySQL Router while bootstrapping"
type: array
items:
type: string
options:
description: "Command line options passed to MySQL Router while running"
type: array
items:
type: string
routingOptions:
description: "Set routing options for the cluster"
type: object
properties:
# naming pattern follows Shell's naming documented at
# https://dev.mysql.com/doc/mysql-shell/8.1/en/innodb-clusterset-router-setroutingoption.html
# ClusterSet-related options and tags currently not supported
invalidated_cluster_policy:
type: string
enum: ["drop_all", "accept_ro"]
stats_updates_frequency:
type: integer
default: 0
minimum: 0
read_only_targets:
type: string
enum: ["all", "read_replicas", "secondaries"]
instanceService:
type: object
description: "Configuration of the Service used by the InnoDB Cluster GR members internally"
properties:
annotations:
type: object
description: "Custom annotations for the Service"
x-kubernetes-preserve-unknown-fields: true
labels:
type: object
description: "Custom labels for the Service"
x-kubernetes-preserve-unknown-fields: true
service:
type: object
description: "Configuration of the Service used by applications connecting to the InnoDB Cluster"
properties:
type:
type: string
enum: ["ClusterIP", "NodePort", "LoadBalancer"]
default: "ClusterIP"
annotations:
type: object
description: "Custom annotations for the Service"
x-kubernetes-preserve-unknown-fields: true
labels:
type: object
description: "Custom labels for the Service"
x-kubernetes-preserve-unknown-fields: true
defaultPort:
type: string
description: "Target for the Service's default (3306) port. If mysql-rw traffic will go to the primary and allow read and write operations, with mysql-ro traffic goes to the replica and allows only read operations, with mysql-rw-split the router's read-write-splitting will be targeted"
enum: ["mysql-rw", "mysql-ro", "mysql-rw-split"]
default: "mysql-rw"
metrics:
type: object
description: "Configuration of a Prometheus-style metrics provider"
required: ["enable", "image"]
properties:
enable:
type: boolean
default: false
description: "Toggle to enable or disable the metrics sidecar"
image:
type: string
description: "Name of an image to be used for the metrics sidecar, if provided metrics will be enabled"
options:
type: array
description: "Options passed to the metrics provider as command line arguments"
items:
type: string
webConfig:
type: string
description: "Name of a ConfigMap with a web.config file, if this option is provided a command line option --web.config.file is added"
tlsSecret:
type: string
description: "Name of a Secret with TLS certificate, key and CA, which will be mounted at /tls into the container an can be used from webConfig"
monitor:
type: boolean
description: "Create a ServiceMonitor for Prometheus Operator"
default: false
monitorSpec:
type: object
x-kubernetes-preserve-unknown-fields: true
description: "Custom configuration for the ServiceMonitor object"
default: {}
# These are only supported for development purpose:
#dbUser:
# type: object
# description: "MySQL user accoutn to be used for collecting metrics"
# properties:
# name:
# type: string
# description: "The account name, host part will always be localhost"
# default: mysqlmetrics
# grants:
# type: array
# description: "GRANTs given to the account"
# default: [ 'PROCESS', 'REPLICATION CLIENT', 'SELECT' ]
# items:
# type: string
# maxConnections:
# type: integer
# default: 3
# description: "Maximum number of connections"
backupProfiles:
type: array
description: "Backup profile specifications for the cluster, which can be referenced from backup schedules and one-off backup jobs"
items:
type: object
required: ["name"]
properties:
name:
type: string
description: "Embedded backup profile, referenced as backupProfileName elsewhere"
podAnnotations:
type: object
x-kubernetes-preserve-unknown-fields: true
podLabels:
type: object
x-kubernetes-preserve-unknown-fields: true
dumpInstance:
type: object
properties:
dumpOptions:
type: object
description: "A dictionary of key-value pairs passed directly to MySQL Shell's DumpInstance()"
x-kubernetes-preserve-unknown-fields: true
storage:
type: object
properties:
ociObjectStorage:
type: object
required: ["bucketName", "credentials"]
properties:
bucketName:
type: string
description: "Name of the OCI bucket where backup is stored"
prefix:
type: string
description: "Path in bucket where backup is stored"
credentials:
type: string
description: "Name of a Secret with data for accessing the bucket"
s3:
type: object
required: ["bucketName", "config"]
properties:
bucketName:
type: string
description: "Name of the S3 bucket where the dump is stored"
prefix:
type: string
description: "Path in the bucket where the dump files are stored"
config:
type: string
description: "Name of a Secret with S3 configuration and credentials"
profile:
type: string
default: ""
description: "Profile being used in configuration files"
endpoint:
type: string
description: "Override endpoint URL"
azure:
type: object
required: ["containerName", "config"]
properties:
containerName:
type: string
description: "Name of the Azure BLOB Storage container where the dump is stored"
prefix:
type: string
description: "Path in the container where the dump files are stored"
config:
type: string
description: "Name of a Secret with Azure BLOB Storage configuration and credentials"
persistentVolumeClaim:
type: object
description : "Specification of the PVC to be used. Used 'as is' in pod executing the backup."
x-kubernetes-preserve-unknown-fields: true
x-kubernetes-preserve-unknown-fields: true
meb:
type: object
description: "MySQL Enterprise Backup"
required: ["storage"]
properties:
storage:
type: object
description: "Configuration of storage for MySQL Enterprise Backup"
properties:
s3:
description: "S3 style storage"
type: object
required: ['region', 'bucket', 'objectKeyPrefix', 'credentials']
properties:
region:
type: string
description: "S3 Region"
bucket:
type: string
description: "S3 Bucket Name"
objectKeyPrefix:
type: string
description: "S3 Object Key Prefix, this will be extended with the name of the actual backup"
credentials:
type: string
description: "Name of a Secret holding iaccessKeyId and secretAccessKey properties"
host:
type: string
description: "Override hostname for S3-compatible backends"
oci:
type: object
description: "Store MySQL Enterprise Backup on OCI Object Storage"
required: ['bucketName', 'prefix', 'namespace', 'credentials']
properties:
bucketName:
type: string
description: "Name of the OCI bucket where backup is stored"
prefix:
type: string
description: "Path in bucket where backup is stored"
namespace:
type: string
description: "OCI Namespace of Object Store"
credentials:
type: string
description: "Name of a Secret with data for accessing the bucket"
extraOptions:
description: "Additonal Command Line Options being passed to MySQL Enterprise Backup"
type: array
items:
type: string
snapshot:
type: object
properties:
storage:
type: object
properties:
ociObjectStorage:
type: object
required: ["bucketName", "credentials"]
properties:
bucketName:
type: string
description: "Bucket name where backup is stored"
prefix:
type: string
description: "Path in bucket where backup is stored"
credentials:
type: string
description: "Name of a Secret with data for accessing the bucket"
s3:
type: object
required: ["bucketName", "config"]
properties:
bucketName:
type: string
description: "Name of the S3 bucket where the dump is stored"
prefix:
type: string
description: "Path in the bucket where the dump files are stored"
config:
type: string
description: "Name of a Secret with S3 configuration and credentials"
profile:
type: string
default: ""
description: "Profile being used in configuration files"
endpoint:
type: string
description: "Override endpoint URL"
azure:
type: object
required: ["containerName", "config"]
properties:
containerName:
type: string
description: "Name of the Azure BLOB Storage container where the dump is stored"
prefix:
type: string
description: "Path in the container where the dump files are stored"
config:
type: string
description: "Name of a Secret with Azure BLOB Storage configuration and credentials"
persistentVolumeClaim:
type: object
description : "Specification of the PVC to be used. Used 'as is' in pod executing the backup."
x-kubernetes-preserve-unknown-fields: true
x-kubernetes-preserve-unknown-fields: true
x-kubernetes-preserve-unknown-fields: true
backupSchedules:
type: array
description: "Schedules for periodically executed backups"
items:
type: object
required: ["name", "schedule"]
x-kubernetes-preserve-unknown-fields: true
properties:
name:
type: string
description: "Name of the backup schedule"
schedule:
type: string
description: "The schedule of the job, syntax as a cron expression"
backupProfileName:
type: string
description: "Name of the backupProfile to be used"
backupProfile:
type: object
description: "backupProfile specification if backupProfileName is not specified"
x-kubernetes-preserve-unknown-fields: true
properties:
podAnnotations:
type: object
x-kubernetes-preserve-unknown-fields: true
podLabels:
type: object
x-kubernetes-preserve-unknown-fields: true
dumpInstance:
type: object
properties:
dumpOptions:
type: object
description: "A dictionary of key-value pairs passed directly to MySQL Shell's DumpInstance()"
x-kubernetes-preserve-unknown-fields: true
storage:
type: object
properties:
ociObjectStorage:
type: object
required: ["bucketName", "credentials"]
properties:
bucketName:
type: string
description: "Name of the OCI Bucket where backup is stored"
prefix:
type: string
description: "Path in bucket where backup is stored"
credentials:
type: string
description: "Name of a Secret with data for accessing the bucket"
s3:
type: object
required: ["bucketName", "config"]
properties:
bucketName:
type: string
description: "Name of the S3 bucket where the dump is stored"
prefix:
type: string
description: "Path in the bucket where the dump files are stored"
config:
type: string
description: "Name of a Secret with S3 configuration and credentials"
profile:
type: string
default: ""
description: "Profile being used in configuration files"
endpoint:
type: string
description: "Override endpoint URL"
azure:
type: object
required: ["containerName", "config"]
properties:
containerName:
type: string
description: "Name of the Azure BLOB Storage container where the dump is stored"
prefix:
type: string
description: "Path in the container where the dump files are stored"
config:
type: string
description: "Name of a Secret with Azure BLOB Storage configuration and credentials"
persistentVolumeClaim:
type: object
description : "Specification of the PVC to be used. Used 'as is' in pod executing the backup."
x-kubernetes-preserve-unknown-fields: true
x-kubernetes-preserve-unknown-fields: true
meb:
type: object
description: "MySQL Enterprise Backup"
required: ["storage"]
properties:
storage:
type: object
description: "Configuration of storage for MySQL Enterprise Backup"
properties:
s3:
description: "S3 style storage"
type: object
required: ['region', 'bucket', 'objectKeyPrefix', 'credentials']
properties:
region:
type: string
description: "S3 Region"
bucket:
type: string
description: "S3 Bucket Name"
objectKeyPrefix:
type: string
description: "S3 Object Key Prefix, this will be extended with the name of the actual backup"
credentials:
type: string
description: "Name of a Secret holding iaccessKeyId and secretAccessKey properties"
host:
type: string
description: "Override hostname for S3-compatible backends"
oci:
type: object
description: "Store MySQL Enterprise Backup on OCI Object Storage"
required: ['bucketName', 'prefix', 'namespace', 'credentials']
properties:
bucketName:
type: string
description: "Name of the OCI bucket where backup is stored"
prefix:
type: string
description: "Path in bucket where backup is stored"
namespace:
type: string
description: "OCI Namespace of Object Store"
credentials:
type: string
description: "Name of a Secret with data for accessing the bucket"
extraOptions:
description: "Additonal Command Line Options being passed to MySQL Enterprise Backup"
type: array
items:
type: string
deleteBackupData:
type: boolean
default: false
description: "Whether to delete the backup data in case the MySQLBackup object created by the job is deleted"
enabled:
type: boolean
default: true
description: "Whether the schedule is enabled or not"
timeZone:
type: string
description: "Timezone for the backup schedule, example: 'America/New_York'"
logs:
type: object
properties:
general:
type: object
properties:
enabled:
type: boolean
default: false
description: "Whether general logging should be enabled"
collect:
type: boolean
default: false
description: "Whether general logging data should be collected. Implies that the logging should be enabled."
error:
type: object
properties:
collect:
type: boolean
default: false
description: "Whether error logging data should be collected. Implies that the logging should be enabled. If enabled the error log will be switched to JSON format output"
verbosity:
type: integer
default: 3
minimum: 1
maximum: 3
description: "Log error verbosity. For details, see the MySQL Server --log-error-verbosity documentation."
slowQuery:
type: object
properties:
enabled:
type: boolean
default: false
description: "Whether slow query logging should be enabled"
longQueryTime:
type: number
minimum: 0
default: 10
description: "Long query time threshold"
collect:
type: boolean
default: false
description: "Whether slow query logging data should be collected. Implies that the logging should be enabled."
collector:
type: object
oneOf:
- required: ["image", "fluentd"]
properties:
image:
type: string
description: "Name of an image, including registry and repository, to be used for the log collector sidecar. If provided it needs to be an image for the configured collector type."
containerName:
type: string
default: "logcollector"
description: "Name of the collector container sidecar"
env:
type: array
items:
type: object
description: "Environment variables to be passed to the image. Definition will be directly copied like podSpec fields are"
x-kubernetes-preserve-unknown-fields: true
fluentd:
type: object
description: "Properties of the fluentd log collector"
oneOf:
- required: ["sinks"]
properties:
generalLog:
type: object
properties:
tag:
type: string
default: ""
description: "Tag for the general log records"
options:
type: object
description: "fluentd specific options for the general log"
x-kubernetes-preserve-unknown-fields: true
errorLog:
type: object
properties:
tag:
type: string
default: ""
description: "Tag for the error log records"
options:
type: object
description: "fluentd specific options for the error log"
x-kubernetes-preserve-unknown-fields: true
slowQueryLog:
type: object
properties:
tag:
type: string
default: ""
description: "Tag for the slow log records"
options:
type: object
description: "fluentd specific options for the slow log"
x-kubernetes-preserve-unknown-fields: true
recordAugmentation:
type: object
properties:
enabled:
type: boolean
default: false
description: "Whether to enable record augmentation with additional data"
labels:
type: array
items:
type: object
required: ["fieldName", "labelName"]
properties:
fieldName:
type: string
description: "Name of the field added to the log record with value from labelName"
labelName:
type: string
description: "Name of the pod label that holds the value to be stored under fieldName in the log record"
annotations:
type: array
items:
type: object
required: ["fieldName", "annotationName"]
properties:
fieldName:
type: string
description: "Name of the field added to the log record with value from annotationName"
annotationName:
type: string
description: "Name of the pod label that holds the value to be stored under fieldName in the log record"
staticFields:
type: array
items:
type: object
required: ["fieldName", "fieldValue"]
properties:
fieldName:
type: string
description: "Name of the field added to the log record with value from fieldValue"
fieldValue:
type: string
description: "Value for the static field with name taken from fieldName"
podFields:
type: array
items:
type: object
required: ["fieldName", "fieldPath"]
properties:
fieldName:
type: string
description: "Name of the field added to the log record with value taken from a field with path stored in fieldPath"
fieldPath:
type: string
description: "Value for the field fieldName. The path should be of the same syntax as the one used for mounting environment variables from field reference - valueFrom.fieldRef.fieldPath . The field will be mounted in the pod as a environment variable, prefixed with a prefix and used then added to the log record. Examples for fieldRef are : spec.nodeName, metadata.namespace, status.podIP, etc."
resourceFields:
type: array
items:
type: object
required: ["fieldName", "containerName", "resource"]
properties:
fieldName:
type: string
description: "Name of the field added to the log record with value taken from a field with path stored in fieldPath"
containerName:
type: string
resource:
type: string
description: "See https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables"
additionalFilterConfiguration:
type: string
description: "Raw configuration of additional Fluentd filters to be added to the configuration file"
sinks:
type: array
items:
type: object
required: ["name", "rawConfig"]
x-kubernetes-preserve-unknown-fields: true
properties:
name:
type: string
description: "Name of the sink. Used only for documentation purposes"
rawConfig:
type: "string"
description: "Raw configuration of the sink"
readReplicas:
type: array
items:
type: object
required: ["name", "baseServerId"]
properties:
name:
type: string
version:
type: string
pattern: '^\d+\.\d+\.\d+(-.+)?'
description: "MySQL Server version"
baseServerId:
type: integer
minimum: 0
maximum: 4294967195
default: 0
description: "Base value for MySQL server_id for instances of the readReplica, if 0 it will be assigned automatically"
datadirVolumeClaimTemplate:
type: object
x-kubernetes-preserve-unknown-fields: true
description: "Template for a PersistentVolumeClaim, to be used as datadir"
mycnf:
type: string
description: "Custom configuration additions for my.cnf"
instances:
type: integer
minimum: 1
maximum: 999
default: 1
description: "Number of MySQL instances for the set of read replica"
podSpec:
type: object
x-kubernetes-preserve-unknown-fields: true
description: "Customization specification to be merged into the InnoDBCluster StatefulSet pod template. IMPORTANT: podSpec is regarded only during StatefulSet buildup and the operator neither checks the values in podSpec nor later watches for changes of this field. Thus if changes are made to this field the InnoDBCluster StatefulSet will not be updated"
podAnnotations:
type: object
x-kubernetes-preserve-unknown-fields: true
podLabels:
type: object
x-kubernetes-preserve-unknown-fields: true
serviceFqdnTemplate:
type: string
description: "Template for a FQDN resolving to the cluster's headless instance Service and individual Pods"
#default: "{service}.{namespace}.svc.{domain}" - We can't set the default as that would override the environment value from the operator
status:
type: object
x-kubernetes-preserve-unknown-fields: true
subresources:
status: {}
additionalPrinterColumns:
- name: Status
type: string
description: Status of the InnoDB Cluster
jsonPath: .status.cluster.status
- name: Online
type: integer
description: Number of ONLINE InnoDB Cluster instances
jsonPath: .status.cluster.onlineInstances
- name: Instances
type: integer
description: Number of InnoDB Cluster instances configured
jsonPath: .spec.instances
- name: Routers
type: integer
description: Number of Router instances configured for the InnoDB Cluster
jsonPath: .spec.router.instances
- name: Type
type: string
description: Type of the cluster in a ClusterSet (PRIMARY, REPLICA, UNKNOWN)
jsonPath: .status.cluster.type
- name: Age
type: date
jsonPath: .metadata.creationTimestamp
scope: Namespaced
names:
kind: InnoDBCluster
listKind: InnoDBClusterList
singular: innodbcluster
plural: innodbclusters
shortNames:
- ic
- ics
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: mysqlbackups.mysql.oracle.com
spec:
group: mysql.oracle.com
scope: Namespaced
names:
kind: MySQLBackup
listKind: MySQLBackupList
singular: mysqlbackup
plural: mysqlbackups
shortNames:
- mbk
versions:
- name: v2
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
required: ["clusterName"]
properties:
clusterName:
type: string
backupProfileName:
type: string
backupProfile:
type: object
description: "backupProfile specification if backupProfileName is not specified"
x-kubernetes-preserve-unknown-fields: true
properties:
podAnnotations:
type: object
x-kubernetes-preserve-unknown-fields: true
podLabels:
type: object
x-kubernetes-preserve-unknown-fields: true
dumpInstance:
type: object
properties:
dumpOptions:
type: object
description: "A dictionary of key-value pairs passed directly to MySQL Shell's DumpInstance()"
x-kubernetes-preserve-unknown-fields: true
storage:
type: object
properties:
ociObjectStorage:
type: object
required: ["bucketName", "credentials"]
properties:
bucketName:
type: string
description: "Name of the OCI bucket where backup is stored"
prefix:
type: string
description: "Path in bucket where backup is stored"
credentials:
type: string
description: "Name of a Secret with data for accessing the bucket"
s3:
type: object
required: ["bucketName", "config"]
properties:
bucketName:
type: string
description: "Name of the S3 bucket where the dump is stored"
prefix:
type: string
description: "Path in the bucket where the dump files are stored"
config:
type: string
description: "Name of a Secret with S3 configuration and credentials"
profile:
type: string
default: ""
description: "Profile being used in configuration files"
endpoint:
type: string
description: "Override endpoint URL"
azure:
type: object
required: ["containerName", "config"]
properties:
containerName:
type: string
description: "Name of the Azure BLOB Storage container where the dump is stored"
prefix:
type: string
description: "Path in the container where the dump files are stored"
config:
type: string
description: "Name of a Secret with Azure BLOB Storage configuration and credentials"
persistentVolumeClaim:
type: object
description : "Specification of the PVC to be used. Used 'as is' in pod executing the backup."
x-kubernetes-preserve-unknown-fields: true
x-kubernetes-preserve-unknown-fields: true
meb:
type: object
description: "MySQL Enterprise Backup"
required: ["storage"]
properties:
storage:
type: object
description: "Configuration of storage for MySQL Enterprise Backup"
properties:
s3:
description: "S3 style storage"
type: object
required: ['region', 'bucket', 'objectKeyPrefix', 'credentials']
properties:
region:
type: string
description: "S3 Region"
bucket:
type: string
description: "S3 Bucket Name"
objectKeyPrefix:
type: string
description: "S3 Object Key Prefix, this will be extended with the name of the actual backup"
credentials:
type: string
description: "Name of a Secret holding iaccessKeyId and secretAccessKey properties"
host:
type: string
description: "Override hostname for S3-compatible backends"
oci:
type: object
description: "Store MySQL Enterprise Backup on OCI Object Storage"
required: ['bucketName', 'prefix', 'namespace', 'credentials']
properties:
bucketName:
type: string
description: "Name of the OCI bucket where backup is stored"
prefix:
type: string
description: "Path in bucket where backup is stored"
namespace:
type: string
description: "OCI Namespace of Object Store"
credentials:
type: string
description: "Name of a Secret with data for accessing the bucket"
extraOptions:
description: "Additonal Command Line Options being passed to MySQL Enterprise Backup"
type: array
items:
type: string
incremental:
type: boolean
default: false
description: "MySQL Enterprise Backup only: Request taking an incremental backup"
incrementalBase:
type: string
description: "MySQL Enterprise Backup only: Base for incremental backup, either last_backup or last_full_backup"
enum: ["last_backup", "last_full_backup"]
default: "last_backup"
addTimestampToBackupDirectory:
type: boolean
default: true
deleteBackupData:
type: boolean
default: false
status:
type: object
properties:
status:
type: string
startTime:
type: string
completionTime:
type: string
elapsedTime:
type: string
output:
type: string
method:
type: string
source:
type: string
bucket:
type: string
ociTenancy:
type: string
container:
type: string
spaceAvailable:
type: string
size:
type: string
message:
type: string
subresources:
status: {}
additionalPrinterColumns:
- name: Cluster
type: string
description: Name of the target cluster
jsonPath: .spec.clusterName
- name: Status
type: string
description: Status of the Backup
jsonPath: .status.status
- name: Output
type: string
description: Name of the produced file/directory
jsonPath: .status.output
- name: Age
type: date
jsonPath: .metadata.creationTimestamp
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: mysqlclustersetfailovers.mysql.oracle.com
spec:
group: mysql.oracle.com
scope: Namespaced
names:
kind: MySQLClusterSetFailover
listKind: MySQLClusterSetFailoverList
singular: mysqlclustersetfailover
plural: mysqlclustersetfailovers
shortNames:
- mfo
versions:
- name: v2
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
required: ["clusterName"]
properties:
clusterName:
type: string
force:
type: boolean
default: false
options:
type: object
properties:
invalidateReplicaClusters:
type: array
items:
type: string
timeout:
type: integer
status:
type: object
properties:
status:
type: string
startTime:
type: string
completionTime:
type: string
elapsedTime:
type: string
subresources:
status: {}
additionalPrinterColumns:
- name: Cluster
type: string
description: Name of the target cluster
jsonPath: .spec.clusterName
- name: Status
type: string
description: Status of the Failover
jsonPath: .status.status
- name: Age
type: date
jsonPath: .metadata.creationTimestamp
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: clusterkopfpeerings.zalando.org
spec:
scope: Cluster
group: zalando.org
names:
kind: ClusterKopfPeering
plural: clusterkopfpeerings
singular: clusterkopfpeering
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
status:
type: object
x-kubernetes-preserve-unknown-fields: true
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: kopfpeerings.zalando.org
spec:
scope: Namespaced
group: zalando.org
names:
kind: KopfPeering
plural: kopfpeerings
singular: kopfpeering
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
status:
type: object
x-kubernetes-preserve-unknown-fields: truecluster yaml
yaml
---
apiVersion: v1
kind: Namespace
metadata:
name: mysql-cluster
---
apiVersion: v1
kind: Secret
metadata:
name: my-cluster-privsql
namespace: mysql-cluster
type: Opaque
stringData:
rootPassword: "Mysql@123"
---
apiVersion: mysql.oracle.com/v2
kind: InnoDBCluster
metadata:
name: my-cluster
namespace: mysql-cluster
spec:
instances: 3
version: "8.0.39"
secretName: my-cluster-privsql
tlsUseSelfSigned: true
mycnf: |
[mysqld]
default_authentication_plugin=caching_sha2_password
max_connect_errors=2000
connect_timeout=10
datadirVolumeClaimTemplate:
storageClassName: "local-path"
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
router:
instances: 2
imageRepository: 10.1.250.75/k8s/mysql
---
apiVersion: v1
kind: Service
metadata:
name: mysql-external-svc
namespace: mysql-cluster
spec:
type: NodePort
ports:
- name: mysql
port: 3306
targetPort: 6446
nodePort: 30306
selector:
"mysql.oracle.com/cluster": my-cluster
"component": mysqlrouter临时恢复连接
kubectl exec -it my-cluster-0 -n mysql-cluster -c mysql -- mysqlsh
\connect root@localhost \sql
sql
-- 1. 查看当前组复制状态(确认当前处于什么状态)
SELECT * FROM performance_schema.replication_group_members;
-- 2. 停止可能挂起的组复制
STOP GROUP_REPLICATION;
-- 3. 开启单节点引导模式(点火)
SET GLOBAL group_replication_bootstrap_group=ON;
-- 4. 重新启动组复制(以自己为首领拉起集群)
START GROUP_REPLICATION;
-- 5. 关闭引导模式(防止后续其他节点启动时发生冲突)
SET GLOBAL group_replication_bootstrap_group=OFF;加入集群
\js
cluster = dba.getCluster()
cluster.rejoinInstance('my-cluster-1.my-cluster-instances.mysql-cluster.svc.cluster.local:3306')
cluster.rejoinInstance('my-cluster-2.my-cluster-instances.mysql-cluster.svc.cluster.local:3306')删除重建
// 1. 强制从集群元数据中移除 1 节点
cluster.removeInstance('my-cluster-1.my-cluster-instances.mysql-cluster.svc.cluster.local:3306', {force: true})
// 2. 作为新节点重新加入集群,自动触发 clone 全量同步
cluster.addInstance('my-cluster-1.my-cluster-instances.mysql-cluster.svc.cluster.local:3306', {recoveryMethod: 'clone'})生产事故复盘:MySQL InnoDB Cluster (K8s) 多数派丢失与 Router 鉴权死锁恢复
一、故障现象与初始状态
- 环境:Kubernetes + Oracle MySQL Operator + MySQL InnoDB Cluster (3 节点) + MySQL Router (2 节点)。
- 初始状态:
my-cluster-0:2/2 Running(唯一存活节点,但失去写能力)。my-cluster-1&my-cluster-2:0/2 Error,删除后重建依然报错,无法自愈。my-cluster-router-*: 报鉴权失败、连接数耗尽,业务全线不可用。
二、核心根因深度剖析
[节点 1、2 崩溃]
↓
[集群丢失多数派 Quorum] → 0 号节点锁死,拒绝接纳变更
↓
[直接删 Pod 未清 PVC] → 新 Pod 带着旧脏数据启动,冲突报错
↓
[清空 PVC 后全新启动] → GTID 为空,无法直接 rejoin,需 remove 后 clone
↓
[Router 不断重试撞库] → 打爆 0 节点 max_connections (1040 报错)
↓
[Router 凭据脱节/复制截断] → 启动脚本隐藏报错,新 Router 持续 CrashLoop- 分布式多数派丢失(Quorum Loss) 3 节点 Group Replication 至少需 2 节点存活才能达成共识。挂掉 2 个后,0 号节点为了防止“脑裂”自动阻塞,丧失单独裁决和接纳新节点的能力。
- K8s StatefulSet 存储机制陷阱 仅删除 Pod 不会删除 PVC。新 Pod 重建后挂载旧 PVC,因状态不一致导致 Rejoin 失败;而彻底删除 PVC 后,新节点成了“空 GTID 节点”,不能直接
rejoinInstance,必须走removeInstance+addInstance(clone)。 - 连接池雪崩(Too many connections 1040) Router 容器死循环重试,耗尽了 MySQL 默认的 151 个连接上限,反过来导致管理员连接和 Shell 执行元数据检查被拦截。
- Router 凭据配置脱节与密码截断陷阱
- Router 配置了
MYSQL_CREATE_ROUTER_USER=0,强行复用固定账号mysqlrouter-OGkz4ugBbg。 - Router 启动脚本中加了
2> /dev/null,掩盖了真实的ERROR 1045。 - 命令行输出无换行符,终端快捷键覆盖粘贴,导致真实密码末尾关键的
6v-ooB+f被吞,引发隐蔽的人为鉴权失败。
- Router 配置了
三、完整恢复操作链路(标准应急手册)
第一阶段:解除 K8s 卡死与存储解绑
当 Pod/PVC 卡在 Terminating 时,强制剥离 Finalizer 释放资源:
bash
# 强行解除 1、2 节点的 Pod 保护钩子(严禁对 0 执行)
kubectl patch pod my-cluster-1 my-cluster-2 -n mysql-cluster -p '{"metadata":{"finalizers":null}}' --type=merge
kubectl delete pod my-cluster-1 my-cluster-2 -n mysql-cluster --force --grace-period=0
# 释放并清空 1、2 节点的旧 PVC,为全量 Clone 腾出干净空间
kubectl patch pvc datadir-my-cluster-1 datadir-my-cluster-2 -n mysql-cluster -p '{"metadata":{"finalizers":null}}' --type=merge
kubectl delete pvc datadir-my-cluster-1 datadir-my-cluster-2 -n mysql-cluster第二阶段:孤存节点(0 号)单机强制点火与扩容连接池
跳过上层 Shell 逻辑,直接通过 SQL 底层解除死锁并抢救主节点:
sql
-- 1. 扩容连接池,防止被重试流量打爆导致运维无法登录 (解决 Error 1040)
SET GLOBAL max_connections = 5000;
-- 2. 强制单节点点火(Bootstrap)拉起 Group Replication
STOP GROUP_REPLICATION;
SET GLOBAL group_replication_bootstrap_group=ON;
START GROUP_REPLICATION;
SET GLOBAL group_replication_bootstrap_group=OFF;
-- 3. 强制解除只读,恢复写能力
SET GLOBAL read_only=OFF;
SET GLOBAL super_read_only=OFF;第三阶段:重建集群元数据与全量克隆(Clone)
针对 PVC 被清空后“GTID 为空”的新节点,先踢出元数据再触发全量同步:
javascript
// 在 0 节点进入 mysqlsh JS 模式
cluster = dba.getCluster()
// 1. 从元数据中强制移除旧拓扑信息
cluster.removeInstance('my-cluster-1.my-cluster-instances.mysql-cluster.svc.cluster.local:3306', {force: true})
cluster.removeInstance('my-cluster-2.my-cluster-instances.mysql-cluster.svc.cluster.local:3306', {force: true})
// 2. 重新纳管并强制走 Clone 插件全量克隆 0 号节点数据
cluster.addInstance('my-cluster-1.my-cluster-instances.mysql-cluster.svc.cluster.local:3306', {recoveryMethod: 'clone'})
cluster.addInstance('my-cluster-2.my-cluster-instances.mysql-cluster.svc.cluster.local:3306', {recoveryMethod: 'clone'})
// 3. 验证集群全绿
cluster.status()第四阶段:Router 鉴权修复与收敛
从存活 Router 提取完整密码,对数据库补齐权限并重启:
bash
# 1. 提取真实密码(注意末尾换行,防止终端截断)
kubectl exec <正常RouterPod> -n mysql-cluster -- cat /.routerpw && echo ""
kubectl exec <正常RouterPod> -n mysql-cluster -- cat /.routeruser && echo ""在 MySQL 中配置带 % 通配网段的真实用户及原生账号:
sql
ALTER USER 'mysqlrouter-OGkz4ugBbg'@'%' IDENTIFIED BY '真实密码完整串';
GRANT ALL PRIVILEGES ON mysql_innodb_cluster_metadata.* TO 'mysqlrouter-OGkz4ugBbg'@'%';
GRANT SELECT ON performance_schema.* TO 'mysqlrouter-OGkz4ugBbg'@'%';
CREATE USER IF NOT EXISTS 'mysqlrouter'@'%' IDENTIFIED BY '真实密码完整串';
ALTER USER 'mysqlrouter'@'%' IDENTIFIED BY '真实密码完整串';
GRANT ALL PRIVILEGES ON mysql_innodb_cluster_metadata.* TO 'mysqlrouter'@'%';
GRANT SELECT ON performance_schema.* TO 'mysqlrouter'@'%';
FLUSH PRIVILEGES;清理旧 Pod 完成滚动更新:
bash
kubectl delete pod -l app.kubernetes.io/name=mysql-router --field-selector=status.phase!=Running -n mysql-cluster四、经验总结与防坑建议(Best Practices)
- 分布式数据库的“删除”与无状态服务完全不同:
- 在 Raft / Paxos 体系下,多数派丢失后切忌无序重启所有节点,必须保护好拥有最新事务的“独苗”,先让它以单机身份成主,再逐步拉从节点。
- 警惕 K8s 终端复制陷阱:
- 查看密码文件(如
cat /.routerpw)时务必加上&& echo "",无换行符的输出极其容易在后续粘贴命令时被吞掉末尾字符。
- 查看密码文件(如
- 生产环境连接数上限(max_connections)预警:
- 默认的 151 极易在集群故障恢复期被重试流量打爆。建议在 Operator 的 CR 模板中显式配置
max_connections >= 2000。
- 默认的 151 极易在集群故障恢复期被重试流量打爆。建议在 Operator 的 CR 模板中显式配置
- 定位脚本报错要善用“剥离重定向”:
- 容器启动脚本(
entrypoint.sh)经常使用2> /dev/null封装错误。遇到反复报通用错误时,进容器内手动执行底层命令,抓取真实的 Exit Code 和错误码。
- 容器启动脚本(