4.3 K8s存储之 Storage Class

kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: standard    #StorageClass是集群级别的对象，不属于任何名称空间
# 指定存储类的供应者
provisioner: kubernetes.io/aws-ebs
parameters:
  type: gp2
# 指定回收策略
reclaimPolicy: Retain
mountOptions:
  - debug

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: slow
provisioner: kubernetes.io/gce-pd
parameters:
  type: pd-standard

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: fast
provisioner: kubernetes.io/gce-pd
parameters:
  type: pd-ssd

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: claim1
spec:
  accessModes:
    - ReadWriteOnce
 storageClassName: fast  # 指定所使用的存储类，此存储类将会自动创建符合要求的PV
 resources:
    requests:
      storage: 30Gi

wget https://raw.githubusercontent.com/kubernetes-incubator/external-storage/master/nfs/deploy/kubernetes/deployment.yaml
sed -i 's/image: quay.io/image: quay-mirror.qiniu.com/g' deployment.yaml 
sed -i 's/example.com/ljzsdut.com/g' deployment.yaml 

...
 volumeMounts:
    - name: export-volume
      mountPath: /export
volumes:
  - name: export-volume
    hostPath:
      path: /tmp/nfs-provisioner
...

args:
  - "-provisioner=example.com/nfs"
...

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-provisioner
---
kind: Service
apiVersion: v1
metadata:
  name: nfs-provisioner
  labels:
    app: nfs-provisioner
spec:
  ports:
    - name: nfs
      port: 2049
    - name: mountd
      port: 20048
    - name: rpcbind
      port: 111
    - name: rpcbind-udp
      port: 111
      protocol: UDP
  selector:
    app: nfs-provisioner
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: nfs-provisioner
spec:
  selector:
    matchLabels:
      app: nfs-provisioner
  replicas: 1
  strategy:
    type: Recreate 
  template:
    metadata:
      labels:
        app: nfs-provisioner
    spec:
      serviceAccount: nfs-provisioner
      containers:
        - name: nfs-provisioner
          image: quay.io/kubernetes_incubator/nfs-provisioner:latest #修改为符合国情的镜像
          ports:
            - name: nfs
              containerPort: 2049
            - name: mountd
              containerPort: 20048
            - name: rpcbind
              containerPort: 111
            - name: rpcbind-udp
              containerPort: 111
              protocol: UDP
          securityContext:
            capabilities:
              add:
                - DAC_READ_SEARCH
                - SYS_RESOURCE
          args:
            # 定义提供者的名称，存储类通过此名称指定提供者。
            - "-provisioner=ljzsdut.com/nfs"   
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: SERVICE_NAME
              value: nfs-provisioner
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            #在/export目录中存储nfs-provisioner实例的状态和nfs数据，该卷需要具有文件系统，不支持NFS
            - name: export-volume  
              mountPath: /export
      volumes:
        - name: export-volume
          hostPath:
            path: /srv  #该目录除了存储nfs-provisioner的一些元数据外还在此目录下创建由sc创建的各个pv的nfs共享目录(数据目录)

$ kubectl create -f {path}/deployment.yaml
service "nfs-provisioner" created
deployment "nfs-provisioner" created

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services", "endpoints"]
    verbs: ["get"]
  - apiGroups: ["extensions"]
    resources: ["podsecuritypolicies"]
    resourceNames: ["nfs-provisioner"]
    verbs: ["use"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-provisioner
     # replace with namespace where provisioner is deployed
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-provisioner
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-provisioner
    # replace with namespace where provisioner is deployed
    namespace: default
roleRef:
  kind: Role
  name: leader-locking-nfs-provisioner
  apiGroup: rbac.authorization.k8s.io

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-storageclass
provisioner: ljzsdut.com/nfs  #上一步中定义的provisioner的名字,必须字段

$ kubectl create -f deploy/kubernetes/class.yaml
storageclass “example-nfs” created

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-pvc
spec:
  accessModes:
  - ReadWriteMany
  storageClassName: nfs-storageclass
  resources:
    requests:
      storage: 1Mi

# This mounts the nfs volume claim into /mnt and continuously
# overwrites /mnt/index.html with the time and hostname of the pod. 
apiVersion: v1
kind: Deployment
metadata:  
  name: busybox-deployment
spec:  
  replicas: 2  
  selector:    
    name: busybox-deployment
  template:    
    metadata:      
      labels:        
        name: busybox-deployment    
    spec:      
      containers:      
      - image: busybox        
        command:          
        - sh          
        - -c          
        - 'while true; do date > /mnt/index.html; hostname >> /mnt/index.html; sleep $(($RANDOM % 5 + 5)); done'        
        imagePullPolicy: IfNotPresent        
        name: busybox        
        volumeMounts:          
        # name must match the volume name below          
        - name: nfs            
          mountPath: "/mnt" 
     volumes:      
     - name: nfs        
       persistentVolumeClaim:          
         claimName: nfs-pvc

$ kubectl create -f {path}/nfs-busybox-deployment.yaml

# provisioner：表示存储分配器，需要根据后端存储的不同而变更；
# reclaimPolicy: 默认即”Delete”，删除pvc后，相应的pv及后端的volume，brick(lvm)等一起删除；设置为”Retain”时则保留数据，需要手工处理
# resturl：heketi API服务提供的url；
# restauthenabled：可选参数，默认值为”false”，heketi服务开启认证时必须设置为”true”；
# restuser：可选参数，开启认证时设置相应用户名；
# secretNamespace：可选参数，开启认证时可以设置为使用持久化存储的namespace；
# secretName：可选参数，开启认证时，需要将heketi服务的认证密码保存在secret资源中；
# clusterid：可选参数，指定集群id，也可以是1个clusterid列表，格式为”id1,id2”；
# volumetype：可选参数，设置卷类型及其参数，如果未分配卷类型，则由分配器决定卷类型；如”volumetype: replicate:3”表示3副本的replicate卷，”volumetype: disperse:4:2”表示disperse卷，其中‘4’是数据，’2’是冗余校验，”volumetype: none”表示distribute分布式卷# 
[root@kubenode1 heketi]# vim gluster-heketi-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: glusterfs
provisioner: kubernetes.io/glusterfs
reclaimPolicy: Delete
parameters:
  resturl: "http://172.30.200.80:8080"
  clusterid: "dae1ab512dfad0001c3911850cecbd61"  #可选
  restauthenabled: "true"
  restuser: "admin"
  secretNamespace: "default"
  secretName: "heketi-secret"   
  #restuserkey: "adminkey"    #明文方式指定密码，不推荐
  volumetype: "replicate:2"
  
---
apiVersion: v1
kind: Secret
metadata:
  name: heketi-secret   #注意name与storageclass资源中的secretName定义一致；
  namespace: default    #注意namespace与storageclass资源中的secretNamespace定义一致
data:
  key: YWRtaW5AMTIz  # base64 encoded password. E.g.: echo -n "mypassword" | base64
type: kubernetes.io/glusterfs  #type必须为“kubernetes.io/glusterfs”

[root@kubenode1 heketi]# kubectl apply -f gluster-heketi-storageclass.yaml  

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mydata-pvc
spec:
  accessModes:
  - ReadWriteMany
  storageClassName: glusterfs
  resources:
    requests:
      storage: 1Mi

$ kubectl create -f claim.yaml

apiVersion: v1
kind: Deployment
metadata:  
  name: busybox-deployment
spec:  
  replicas: 2  
  selector:    
    name: busybox-deployment
  template:    
    metadata:      
      labels:        
        name: busybox-deployment    
    spec:      
      containers:      
      - image: busybox        
        command:          
        - sh          
        - -c          
        - 'while true; do date > /mnt/index.html; hostname >> /mnt/index.html; sleep $(($RANDOM % 5 + 5)); done'        
        imagePullPolicy: IfNotPresent        
        name: busybox        
        volumeMounts:          
        - name: data           
          mountPath: "/mnt" 
     volumes:      
     - name: data        
       persistentVolumeClaim:          
         claimName: mydata-pvc

$ kubectl create -f nfs-busybox-deployment.yaml