Ceph Installation

On openstack-staging machine, make sure you have have ansible 2.10 version been installed.

ssh-keygen -t rsa 
ssh-copy-id root@openstack-ceph01
ssh-copy-id root@openstack-ceph02
ssh-copy-id root@openstack-ceph03

Install anisble

apt-get install -y software-properties-common git cowsay
apt-add-repository -y ppa:ansible/ansible
apt-get update && sudo apt-get install -y ansible

Download ansible-ceph

git clone https://github.com/ceph/ceph-ansible.git
cd ceph-ansible
git checkout -b origin/stable-6.0
pip3 install -r requirements.txt

Create host file

vim /etc/ansible/hosts
[mons]
openstack-ceph01
openstack-ceph02
openstack-ceph03
[osds]
openstack-ceph01
openstack-ceph02
openstack-ceph03
[rgws]
openstack-ceph01
openstack-ceph02
openstack-ceph03
[clients]
openstack-ceph01
openstack-ceph02
openstack-ceph03
[mgrs]
openstack-ceph01
openstack-ceph02
openstack-ceph03
[monitoring]
openstack-ceph01
openstack-ceph02
openstack-ceph03

Check connectivity

ansible all -m ping

Change file name

cp site.yml.sample site.yml
cp group_vars/osds.yml.sample group_vars/osds.yml
cp group_vars/clients.yml.sample group_vars/clients.yml
cp group_vars/mons.yml.sample group_vars/mons.yml
cp group_vars/mgrs.yml.sample group_vars/mgrs.yml
cp group_vars/all.yml.sample group_vars/all.yml

Now prepare group_vars/all.yml file

Summary

root@openstack-staging:/home/kevin/ceph-ansible# grep -v '^\s*$\|^\s*\#'  group_vars/all.yml
---
dummy:
ceph_release_num:
pacific: 16
cluster: ceph
ntp_service_enabled: true
ntp_daemon_type: chronyd
ceph_repository_type: repository
ceph_origin: repository
ceph_repository: community
ceph_mirror: https://download.ceph.com
ceph_stable_key: https://download.ceph.com/keys/release.asc
ceph_stable_release: pacific
ceph_stable_repo: "https://download.ceph.com/debian-pacific"
cephx: true
rbd_cache: "true"
rbd_cache_writethrough_until_flush: "true"
rbd_concurrent_management_ops: 20
monitor_interface: ens160
ip_version: ipv4
public_network: 10.196.24.0/24
cluster_network: "192.168.0.0/24"
osd_mkfs_type: xfs
osd_mkfs_options_xfs: -f -i size=2048
osd_mount_options_xfs: noatime,largeio,inode64,swalloc
osd_objectstore: bluestore
radosgw_civetweb_port: 8080
radosgw_interface: ens160
ceph_docker_image: "ceph/daemon"
ceph_docker_image_tag: latest-pacific
containerized_deployment: True
dashboard_enabled: True
dashboard_protocol: https
dashboard_port: 8443
dashboard_admin_user: admin
dashboard_admin_password: <password>
grafana_admin_user: admin
grafana_admin_password: <password>
grafana_container_image: "docker.io/grafana/grafana:6.7.4"
grafana_uid: 472
grafana_datasource: Dashboard
grafana_dashboards_path: "/etc/grafana/dashboards/ceph-dashboard"
grafana_dashboard_version: master
grafana_dashboard_files:
- ceph-cluster.json
- cephfs-overview.json
- host-details.json
- hosts-overview.json
- osd-device-details.json
- osds-overview.json
- pool-detail.json
- pool-overview.json
- radosgw-detail.json
- radosgw-overview.json
- rbd-overview.json
grafana_port: 3000
prometheus_port: 9092

Full group_vars/all.yml configuration file

root@openstack-staging:/home/kevin/ceph-ansible# cat group_vars/all.yml---
# Variables here are applicable to all host groups NOT roles
# This sample file generated by generate_group_vars_sample.sh# Dummy variable to avoid error because ansible does not recognize the
# file as a good configuration file when no variable in it.
dummy:
# You can override vars by using host or group vars###########
# GENERAL #
###########
######################################
# Releases name to number dictionary #
######################################
ceph_release_num:
# dumpling: 0.67
# emperor: 0.72
# firefly: 0.80
# giant: 0.87
# hammer: 0.94
# infernalis: 9
# jewel: 10
# kraken: 11
# luminous: 12
# mimic: 13
# nautilus: 14
# octopus: 15
pacific: 16
# quincy: 17
# dev: 99
# The 'cluster' variable determines the name of the cluster.
# Changing the default value to something else means that you will
# need to change all the command line calls as well, for example if
# your cluster name is 'foo':
# "ceph health" will become "ceph --cluster foo health"
#
# An easier way to handle this is to use the environment variable CEPH_ARGS
# So run: "export CEPH_ARGS="--cluster foo"
# With that you will be able to run "ceph health" normally
cluster: ceph
# Inventory host group variables
#mon_group_name: mons
#osd_group_name: osds
#rgw_group_name: rgws
#mds_group_name: mdss
#nfs_group_name: nfss
#rbdmirror_group_name: rbdmirrors
#client_group_name: clients
#iscsi_gw_group_name: iscsigws
#mgr_group_name: mgrs
#rgwloadbalancer_group_name: rgwloadbalancers
#monitoring_group_name: monitoring
# If configure_firewall is true, then ansible will try to configure the
# appropriate firewalling rules so that Ceph daemons can communicate
# with each others.
#configure_firewall: True
# Open ports on corresponding nodes if firewall is installed on it
#ceph_mon_firewall_zone: public
#ceph_mgr_firewall_zone: public
#ceph_osd_firewall_zone: public
#ceph_rgw_firewall_zone: public
#ceph_mds_firewall_zone: public
#ceph_nfs_firewall_zone: public
#ceph_rbdmirror_firewall_zone: public
#ceph_iscsi_firewall_zone: public
#ceph_dashboard_firewall_zone: public
#ceph_rgwloadbalancer_firewall_zone: public
############
# PACKAGES #
############
#debian_package_dependencies: []
#centos_package_dependencies:
# - epel-release
# - "{{ (ansible_facts['distribution_major_version'] is version('8', '>=')) | ternary('python3-libselinux', 'libselinux-python') }}"
#redhat_package_dependencies: []#suse_package_dependencies: []# Whether or not to install the ceph-test package.
#ceph_test: false
# Enable the ntp service by default to avoid clock skew on ceph nodes
# Disable if an appropriate NTP client is already installed and configured
ntp_service_enabled: true
# Set type of NTP client daemon to use, valid entries are chronyd, ntpd or timesyncd
ntp_daemon_type: chronyd
# This variable determines if ceph packages can be updated. If False, the
# package resources will use "state=present". If True, they will use
# "state=latest".
#upgrade_ceph_packages: False
#ceph_use_distro_backports: false # DEBIAN ONLY
#ceph_directories_mode: "0755"
###########
# INSTALL #
###########
ceph_repository_type: repository
# ORIGIN SOURCE
#
# Choose between:
# - 'repository' means that you will get ceph installed through a new repository. Later below choose between 'community', 'rhcs', 'dev' or 'obs'
# - 'distro' means that no separate repo file will be added
# you will get whatever version of Ceph is included in your Linux distro.
# 'local' means that the ceph binaries will be copied over from the local machine
ceph_origin: repository
#valid_ceph_origins:
# - repository
# - distro
# - local
ceph_repository: community
#valid_ceph_repository:
# - community
# - rhcs
# - dev
# - uca
# - custom
# - obs
# REPOSITORY: COMMUNITY VERSION
#
# Enabled when ceph_repository == 'community'
#
ceph_mirror: https://download.ceph.com
ceph_stable_key: https://download.ceph.com/keys/release.asc
ceph_stable_release: pacific
ceph_stable_repo: "https://download.ceph.com/debian-pacific"
#nfs_ganesha_stable: true # use stable repos for nfs-ganesha
#nfs_ganesha_stable_branch: V3.5-stable
#nfs_ganesha_stable_deb_repo: "{{ ceph_mirror }}/nfs-ganesha/deb-{{ nfs_ganesha_stable_branch }}/{{ ceph_stable_release }}"
# Use the option below to specify your applicable package tree, eg. when using non-LTS Ubuntu versions
# # for a list of available Debian distributions, visit http://download.ceph.com/debian-{{ ceph_stable_release }}/dists/
# for more info read: https://github.com/ceph/ceph-ansible/issues/305
#ceph_stable_distro_source: "{{ ansible_facts['distribution_release'] }}"
# REPOSITORY: RHCS VERSION RED HAT STORAGE (from 5.0)
#
# Enabled when ceph_repository == 'rhcs'
#
# This version is supported on RHEL 8
#
#ceph_rhcs_version: "{{ ceph_stable_rh_storage_version | default(5) }}"
#valid_ceph_repository_type:
# - cdn
# - iso
#ceph_rhcs_iso_path: "{{ ceph_stable_rh_storage_iso_path | default('') }}"
#ceph_rhcs_mount_path: "{{ ceph_stable_rh_storage_mount_path | default('/tmp/rh-storage-mount') }}"
#ceph_rhcs_repository_path: "{{ ceph_stable_rh_storage_repository_path | default('/tmp/rh-storage-repo') }}" # where to copy iso's content
# REPOSITORY: UBUNTU CLOUD ARCHIVE
#
# Enabled when ceph_repository == 'uca'
#
# This allows the install of Ceph from the Ubuntu Cloud Archive. The Ubuntu Cloud Archive
# usually has newer Ceph releases than the normal distro repository.
#
#
#ceph_stable_repo_uca: "http://ubuntu-cloud.archive.canonical.com/ubuntu"
#ceph_stable_openstack_release_uca: queens
#ceph_stable_release_uca: "{{ ansible_facts['distribution_release'] }}-updates/{{ ceph_stable_openstack_release_uca }}"
# REPOSITORY: openSUSE OBS
#
# Enabled when ceph_repository == 'obs'
#
# This allows the install of Ceph from the openSUSE OBS repository. The OBS repository
# usually has newer Ceph releases than the normal distro repository.
#
#
#ceph_obs_repo: "https://download.opensuse.org/repositories/filesystems:/ceph:/{{ ceph_stable_release }}/openSUSE_Leap_{{ ansible_facts['distribution_version'] }}/"
# REPOSITORY: DEV
#
# Enabled when ceph_repository == 'dev'
#
#ceph_dev_branch: master # development branch you would like to use e.g: master, wip-hack
#ceph_dev_sha1: latest # distinct sha1 to use, defaults to 'latest' (as in latest built)
#nfs_ganesha_dev: false # use development repos for nfs-ganesha# Set this to choose the version of ceph dev libraries used in the nfs-ganesha packages from shaman
# flavors so far include: ceph_master, ceph_jewel, ceph_kraken, ceph_luminous
#nfs_ganesha_flavor: "ceph_master"
#ceph_iscsi_config_dev: true # special repo for deploying iSCSI gateways# REPOSITORY: CUSTOM
#
# Enabled when ceph_repository == 'custom'
#
# Use a custom repository to install ceph. For RPM, ceph_custom_repo should be
# a URL to the .repo file to be installed on the targets. For deb,
# ceph_custom_repo should be the URL to the repo base.
#
#ceph_custom_key: https://server.domain.com/ceph-custom-repo/key.asc
#ceph_custom_repo: https://server.domain.com/ceph-custom-repo
# ORIGIN: LOCAL CEPH INSTALLATION
#
# Enabled when ceph_repository == 'local'
#
# Path to DESTDIR of the ceph install
#ceph_installation_dir: "/path/to/ceph_installation/"
# Whether or not to use installer script rundep_installer.sh
# This script takes in rundep and installs the packages line by line onto the machine
# If this is set to false then it is assumed that the machine ceph is being copied onto will already have
# all runtime dependencies installed
#use_installer: false
# Root directory for ceph-ansible
#ansible_dir: "/path/to/ceph-ansible"
######################
# CEPH CONFIGURATION #
######################
## Ceph options
#
# Each cluster requires a unique, consistent filesystem ID. By
# default, the playbook generates one for you.
# If you want to customize how the fsid is
# generated, you may find it useful to disable fsid generation to
# avoid cluttering up your ansible repo. If you set `generate_fsid` to
# false, you *must* generate `fsid` in another way.
# ACTIVATE THE FSID VARIABLE FOR NON-VAGRANT DEPLOYMENT
#fsid: "{{ cluster_uuid.stdout }}"
#generate_fsid: true
#ceph_conf_key_directory: /etc/ceph#ceph_uid: "{{ '64045' if not containerized_deployment | bool and ansible_facts['os_family'] == 'Debian' else '167' }}"# Permissions for keyring files in /etc/ceph
#ceph_keyring_permissions: '0600'
cephx: true## Client options
#
rbd_cache: "true"
rbd_cache_writethrough_until_flush: "true"
rbd_concurrent_management_ops: 20
#rbd_client_directories: true # this will create rbd_client_log_path and rbd_client_admin_socket_path directories with proper permissions# Permissions for the rbd_client_log_path and
# rbd_client_admin_socket_path. Depending on your use case for Ceph
# you may want to change these values. The default, which is used if
# any of the variables are unset or set to a false value (like `null`
# or `false`) is to automatically determine what is appropriate for
# the Ceph version with non-OpenStack workloads -- ceph:ceph and 0770
# for infernalis releases, and root:root and 1777 for pre-infernalis
# releases.
#
# For other use cases, including running Ceph with OpenStack, you'll
# want to set these differently:
#
# For OpenStack on RHEL, you'll want:
# rbd_client_directory_owner: "qemu"
# rbd_client_directory_group: "libvirtd" (or "libvirt", depending on your version of libvirt)
# rbd_client_directory_mode: "0755"
#
# For OpenStack on Ubuntu or Debian, set:
# rbd_client_directory_owner: "libvirt-qemu"
# rbd_client_directory_group: "kvm"
# rbd_client_directory_mode: "0755"
#
# If you set rbd_client_directory_mode, you must use a string (e.g.,
# 'rbd_client_directory_mode: "0755"', *not*
# 'rbd_client_directory_mode: 0755', or Ansible will complain: mode
# must be in octal or symbolic form
#rbd_client_directory_owner: ceph
#rbd_client_directory_group: ceph
#rbd_client_directory_mode: "0770"
#rbd_client_log_path: /var/log/ceph
#rbd_client_log_file: "{{ rbd_client_log_path }}/qemu-guest-$pid.log" # must be writable by QEMU and allowed by SELinux or AppArmor
#rbd_client_admin_socket_path: /var/run/ceph # must be writable by QEMU and allowed by SELinux or AppArmor
## Monitor options
#
# You must define either monitor_interface, monitor_address or monitor_address_block.
# These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
# Eg. If you want to specify for each monitor which address the monitor will bind to you can set it in your **inventory host file** by using 'monitor_address' variable.
# Preference will go to monitor_address if both monitor_address and monitor_interface are defined.
monitor_interface: ens160
#monitor_address:
#monitor_address_block: subnet
# set to either ipv4 or ipv6, whichever your network is using
ip_version: ipv4
#mon_host_v1:
# enabled: True
# suffix: ':6789'
#mon_host_v2:
# suffix: ':3300'
#enable_ceph_volume_debug: False##########
# CEPHFS #
##########
# When pg_autoscale_mode is set to True, you must add the target_size_ratio key with a correct value
# `pg_num` and `pgp_num` keys will be ignored, even if specified.
# eg:
# cephfs_data_pool:
# name: "{{ cephfs_data if cephfs_data is defined else 'cephfs_data' }}"
# target_size_ratio: 0.2
#cephfs: cephfs # name of the ceph filesystem
#cephfs_data_pool:
# name: "{{ cephfs_data if cephfs_data is defined else 'cephfs_data' }}"
#cephfs_metadata_pool:
# name: "{{ cephfs_metadata if cephfs_metadata is defined else 'cephfs_metadata' }}"
#cephfs_pools:
# - "{{ cephfs_data_pool }}"
# - "{{ cephfs_metadata_pool }}"
## OSD options
#
#lvmetad_disabled: false
#is_hci: false
#hci_safety_factor: 0.2
#non_hci_safety_factor: 0.7
#osd_memory_target: 4294967296
#journal_size: 5120 # OSD journal size in MB
#block_db_size: -1 # block db size in bytes for the ceph-volume lvm batch. -1 means use the default of 'as big as possible'.
public_network: 10.196.24.0/24
cluster_network: "192.168.0.0/24"
osd_mkfs_type: xfs
osd_mkfs_options_xfs: -f -i size=2048
osd_mount_options_xfs: noatime,largeio,inode64,swalloc
osd_objectstore: bluestore
# Any device containing these patterns in their path will be excluded.
#osd_auto_discovery_exclude: "dm-*|loop*|md*|rbd*"
# xattrs. by default, 'filestore xattr use omap' is set to 'true' if
# 'osd_mkfs_type' is set to 'ext4'; otherwise it isn't set. This can
# be set to 'true' or 'false' to explicitly override those
# defaults. Leave it 'null' to use the default for your chosen mkfs
# type.
#filestore_xattr_use_omap: null
## MDS options
#
#mds_max_mds: 1
## Rados Gateway options
#
#radosgw_frontend_type: beast # For additional frontends see: https://docs.ceph.com/en/latest/radosgw/frontends/
radosgw_civetweb_port: 8080
#radosgw_civetweb_num_threads: 512
#radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
# For additional civetweb configuration options available such as logging,
# keepalive, and timeout settings, please see the civetweb docs at
# https://github.com/civetweb/civetweb/blob/master/docs/UserManual.md
#radosgw_frontend_port: "{{ radosgw_civetweb_port if radosgw_frontend_type == 'civetweb' else '8080' }}"
# The server private key, public certificate and any other CA or intermediate certificates should be in one file, in PEM format.
#radosgw_frontend_ssl_certificate: ""
#radosgw_frontend_ssl_certificate_data: "" # certificate contents to be written to path defined by radosgw_frontend_ssl_certificate
#radosgw_frontend_options: "{{ radosgw_civetweb_options if radosgw_frontend_type == 'civetweb' else '' }}"
#radosgw_thread_pool_size: 512
# You must define either radosgw_interface, radosgw_address.
# These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
# Eg. If you want to specify for each radosgw node which address the radosgw will bind to you can set it in your **inventory host file** by using 'radosgw_address' variable.
# Preference will go to radosgw_address if both radosgw_address and radosgw_interface are defined.
radosgw_interface: ens160
#radosgw_address: x.x.x.x
#radosgw_address_block: subnet
#radosgw_keystone_ssl: false # activate this when using keystone PKI keys
#radosgw_num_instances: 1
# Rados Gateway options
#email_address: foo@bar.com
## Testing mode
# enable this mode _only_ when you have a single node
# if you don't want it keep the option commented
#common_single_host_mode: true
## Handlers - restarting daemons after a config change
# if for whatever reasons the content of your ceph configuration changes
# ceph daemons will be restarted as well. At the moment, we can not detect
# which config option changed so all the daemons will be restarted. Although
# this restart will be serialized for each node, in between a health check
# will be performed so we make sure we don't move to the next node until
# ceph is not healthy
# Obviously between the checks (for monitors to be in quorum and for osd's pgs
# to be clean) we have to wait. These retries and delays can be configurable
# for both monitors and osds.
#
# Monitor handler checks
#handler_health_mon_check_retries: 10
#handler_health_mon_check_delay: 20
#
# OSD handler checks
#handler_health_osd_check_retries: 40
#handler_health_osd_check_delay: 30
#handler_health_osd_check: true
#
# MDS handler checks
#handler_health_mds_check_retries: 5
#handler_health_mds_check_delay: 10
#
# RGW handler checks
#handler_health_rgw_check_retries: 5
#handler_health_rgw_check_delay: 10
# NFS handler checks
#handler_health_nfs_check_retries: 5
#handler_health_nfs_check_delay: 10
# RBD MIRROR handler checks
#handler_health_rbd_mirror_check_retries: 5
#handler_health_rbd_mirror_check_delay: 10
# MGR handler checks
#handler_health_mgr_check_retries: 5
#handler_health_mgr_check_delay: 10
## health mon/osds check retries/delay:#health_mon_check_retries: 20
#health_mon_check_delay: 10
#health_osd_check_retries: 20
#health_osd_check_delay: 10
###############
# NFS-GANESHA #
###############
#
# Access type options
#
# Enable NFS File access
# If set to true, then ganesha is set up to export the root of the
# Ceph filesystem, and ganesha's attribute and directory caching is disabled
# as much as possible since libcephfs clients also caches the same
# information.
#
# Set this to true to enable File access via NFS. Requires an MDS role.
#nfs_file_gw: false
# Set this to true to enable Object access via NFS. Requires an RGW role.
#nfs_obj_gw: "{{ False if groups.get(mon_group_name, []) | length == 0 else True }}"
#############
# MULTISITE #
#############
# Changing this value allows multisite code to run
#rgw_multisite: false
# If the desired multisite configuration involves only one realm, one zone group and one zone (per cluster), then the multisite variables can be set here.
# Please see README-MULTISITE.md for more information.
#
# If multiple realms or multiple zonegroups or multiple zones need to be created on a cluster then,
# the multisite config variables should be editted in their respective zone .yaml file and realm .yaml file.
# See README-MULTISITE.md for more information.
# The following Multi-site related variables should be set by the user.
#
# rgw_zone is set to "default" to enable compression for clusters configured without rgw multi-site
# If multisite is configured, rgw_zone should not be set to "default".
#
#rgw_zone: default
#rgw_zonemaster: true
#rgw_zonesecondary: false
#rgw_zonegroup: solarsystem # should be set by the user
#rgw_zonegroupmaster: true
#rgw_zone_user: zone.user
#rgw_zone_user_display_name: "Zone User"
#rgw_realm: milkyway # should be set by the user
#rgw_multisite_proto: "http"
#system_access_key: 6kWkikvapSnHyE22P7nO # should be re-created by the user
#system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt # should be re-created by the user
# Multi-site remote pull URL variables
#rgw_pull_port: "{{ radosgw_frontend_port }}"
#rgw_pull_proto: "http" # should be the same as rgw_multisite_proto for the master zone cluster
#rgw_pullhost: localhost # rgw_pullhost only needs to be declared if there is a zone secondary.
###################
# CONFIG OVERRIDE #
###################
# Ceph configuration file override.
# This allows you to specify more configuration options
# using an INI style format.
#
# When configuring RGWs, make sure you use the form [client.rgw.*]
# instead of [client.radosgw.*].
# For more examples check the profiles directory of https://github.com/ceph/ceph-ansible.
#
# The following sections are supported: [global], [mon], [osd], [mds], [client]
#
# Example:
# ceph_conf_overrides:
# global:
# foo: 1234
# bar: 5678
# "client.rgw.{{ hostvars[groups.get(rgw_group_name)[0]]['ansible_facts']['hostname'] }}":
# rgw_zone: zone1
#
#ceph_conf_overrides: {}
#############
# OS TUNING #
#############
#disable_transparent_hugepage: "{{ false if osd_objectstore == 'bluestore' else true }}"
#os_tuning_params:
# - { name: fs.file-max, value: 26234859 }
# - { name: vm.zone_reclaim_mode, value: 0 }
# - { name: vm.swappiness, value: 10 }
# - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
# Set this to a byte value (e.g. 134217728)
# A value of 0 will leave the package default.
#ceph_tcmalloc_max_total_thread_cache: 0
##########
# DOCKER #
##########
ceph_docker_image: "ceph/daemon"
ceph_docker_image_tag: latest-pacific
#ceph_docker_registry: docker.io
#ceph_docker_registry_auth: false
#ceph_docker_registry_username:
#ceph_docker_registry_password:
#ceph_docker_http_proxy:
#ceph_docker_https_proxy:
#ceph_docker_no_proxy: "localhost,127.0.0.1"
## Client only docker image - defaults to {{ ceph_docker_image }}
#ceph_client_docker_image: "{{ ceph_docker_image }}"
#ceph_client_docker_image_tag: "{{ ceph_docker_image_tag }}"
#ceph_client_docker_registry: "{{ ceph_docker_registry }}"
containerized_deployment: True
#container_binary: ceph/daemon:latest-octopus
#timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}"
# this is only here for usage with the rolling_update.yml playbook
# do not ever change this here
#rolling_update: false
#####################
# Docker pull retry #
#####################
#docker_pull_retry: 3
#docker_pull_timeout: "300s"
#############
# OPENSTACK #
#############
#openstack_config: false
# When pg_autoscale_mode is set to True, you must add the target_size_ratio key with a correct value
# `pg_num` and `pgp_num` keys will be ignored, even if specified.
# eg:
# openstack_glance_pool:
# name: "images"
# rule_name: "my_replicated_rule"
# application: "rbd"
# pg_autoscale_mode: False
# pg_num: 16
# pgp_num: 16
# target_size_ratio: 0.2
#openstack_glance_pool:
# name: "images"
# application: "rbd"
#openstack_cinder_pool:
# name: "volumes"
# application: "rbd"
#openstack_nova_pool:
# name: "vms"
# application: "rbd"
#openstack_cinder_backup_pool:
# name: "backups"
# application: "rbd"
#openstack_gnocchi_pool:
# name: "metrics"
# application: "rbd"
#openstack_cephfs_data_pool:
# name: "manila_data"
# application: "cephfs"
#openstack_cephfs_metadata_pool:
# name: "manila_metadata"
# application: "cephfs"
#openstack_pools:
# - "{{ openstack_glance_pool }}"
# - "{{ openstack_cinder_pool }}"
# - "{{ openstack_nova_pool }}"
# - "{{ openstack_cinder_backup_pool }}"
# - "{{ openstack_gnocchi_pool }}"
# - "{{ openstack_cephfs_data_pool }}"
# - "{{ openstack_cephfs_metadata_pool }}"
# The value for 'key' can be a pre-generated key,
# e.g key: "AQDC2UxZH4yeLhAAgTaZb+4wDUlYOsr1OfZSpQ=="
# By default, keys will be auto-generated.
#
#openstack_keys:
# - { name: client.glance, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_cinder_pool.name }}, profile rbd pool={{ openstack_glance_pool.name }}"}, mode: "0600" }
# - { name: client.cinder, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_cinder_pool.name }}, profile rbd pool={{ openstack_nova_pool.name }}, profile rbd pool={{ openstack_glance_pool.name }}"}, mode: "0600" }
# - { name: client.cinder-backup, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_cinder_backup_pool.name }}"}, mode: "0600" }
# - { name: client.gnocchi, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_gnocchi_pool.name }}"}, mode: "0600", }
# - { name: client.openstack, caps: { mon: "profile rbd", osd: "profile rbd pool={{ openstack_glance_pool.name }}, profile rbd pool={{ openstack_nova_pool.name }}, profile rbd pool={{ openstack_cinder_pool.name }}, profile rbd pool={{ openstack_cinder_backup_pool.name }}"}, mode: "0600" }
#############
# DASHBOARD #
#############
dashboard_enabled: True
# Choose http or https
# For https, you should set dashboard.crt/key and grafana.crt/key
# If you define the dashboard_crt and dashboard_key variables, but leave them as '',
# then we will autogenerate a cert and keyfile
dashboard_protocol: https
dashboard_port: 8443
dashboard_admin_user: admin
#dashboard_admin_user_ro: false
# This variable must be set with a strong custom password when dashboard_enabled is True
dashboard_admin_password: <Password>
# We only need this for SSL (https) connections
#dashboard_crt: ''
#dashboard_key: ''
#dashboard_tls_external: false
#dashboard_grafana_api_no_ssl_verify: "{{ true if dashboard_protocol == 'https' and not grafana_crt and not grafana_key else false }}"
#dashboard_rgw_api_user_id: ceph-dashboard
#dashboard_rgw_api_admin_resource:
#dashboard_rgw_api_no_ssl_verify: False
#dashboard_frontend_vip: ''
#prometheus_frontend_vip: ''
#alertmanager_frontend_vip: ''
#node_exporter_container_image: "docker.io/prom/node-exporter:v0.17.0"
#node_exporter_port: 9100
grafana_admin_user: admin
# This variable must be set with a strong custom password when dashboard_enabled is True
grafana_admin_password: <Password>
# We only need this for SSL (https) connections
#grafana_crt: ''
#grafana_key: ''
# When using https, please fill with a hostname for which grafana_crt is valid.
#grafana_server_fqdn: ''
grafana_container_image: "docker.io/grafana/grafana:6.7.4"
#grafana_container_cpu_period: 100000
#grafana_container_cpu_cores: 2
# container_memory is in GB
#grafana_container_memory: 4
grafana_uid: 472
grafana_datasource: Dashboard
grafana_dashboards_path: "/etc/grafana/dashboards/ceph-dashboard"
grafana_dashboard_version: master
grafana_dashboard_files:
- ceph-cluster.json
- cephfs-overview.json
- host-details.json
- hosts-overview.json
- osd-device-details.json
- osds-overview.json
- pool-detail.json
- pool-overview.json
- radosgw-detail.json
- radosgw-overview.json
- rbd-overview.json
#grafana_plugins:
# - vonage-status-panel
# - grafana-piechart-panel
#grafana_allow_embedding: True
grafana_port: 3000
#grafana_conf_overrides: {}
#prometheus_container_image: "docker.io/prom/prometheus:v2.7.2"
#prometheus_container_cpu_period: 100000
#prometheus_container_cpu_cores: 2
# container_memory is in GB
#prometheus_container_memory: 4
#prometheus_data_dir: /var/lib/prometheus
#prometheus_conf_dir: /etc/prometheus
#prometheus_user_id: '65534' # This is the UID used by the prom/prometheus container image
prometheus_port: 9092
#prometheus_conf_overrides: {}
# Uncomment out this variable if you need to customize the retention period for prometheus storage.
# set it to '30d' if you want to retain 30 days of data.
#prometheus_storage_tsdb_retention_time: 15d
#alertmanager_container_image: "docker.io/prom/alertmanager:v0.16.2"
#alertmanager_container_cpu_period: 100000
#alertmanager_container_cpu_cores: 2
# container_memory is in GB
#alertmanager_container_memory: 4
#alertmanager_data_dir: /var/lib/alertmanager
#alertmanager_conf_dir: /etc/alertmanager
#alertmanager_port: 9093
#alertmanager_cluster_port: 9094
#alertmanager_conf_overrides: {}
# igw
#
# `igw_network` variable is intended for allowing dashboard deployment with iSCSI node not residing in the same subnet than what is defined in `public_network`.
# For example:
# If the ceph public network is 2a00:8a60:1:c301::/64 and the iSCSI Gateway resides
# at a dedicated gateway network (2a00:8a60:1:c300::/64) (With routing between those networks).
# It means "{{ hostvars[item]['ansible_facts']['all_ipv4_addresses'] | ips_in_ranges(public_network.split(',')) | last | ipwrap }}" will be empty.
# As a consequence, this prevent from deploying dashboard with iSCSI node when it reside in a subnet different than `public_network`.
# Using `igw_network` make it possible, set it with the subnet used by your iSCSI node.
#igw_network: "{{ public_network }}"
##################################
# DEPRECIATED iSCSI TARGET SETUP #
##################################
# WARNING ## The following values are depreciated. To setup targets, gateways, LUNs, and
# clients you should use gwcli or dashboard. If the following values are set,
# the old ceph-iscsi-config/ceph-iscsi-cli packages will be used.
# Specify the iqn for ALL gateways. This iqn is shared across the gateways, so an iscsi
# client sees the gateway group as a single storage subsystem.
#gateway_iqn: ""
# gateway_ip_list provides a list of the IP Addrresses - one per gateway - that will be used
# as an iscsi target portal ip. The list must be comma separated - and the order determines
# the sequence of TPG's within the iscsi target across each gateway. Once set, additional
# gateways can be added, but the order must *not* be changed.
#gateway_ip_list: 0.0.0.0
# rbd_devices defines the images that should be created and exported from the iscsi gateways.
# If the rbd does not exist, it will be created for you. In addition you may increase the
# size of rbd's by changing the size parameter and rerunning the playbook. A size value lower
# than the current size of the rbd is ignored.
#
# the 'host' parameter defines which of the gateway nodes should handle the physical
# allocation/expansion or removal of the rbd
# to remove an image, simply use a state of 'absent'. This will first check the rbd is not allocated
# to any client, and the remove it from LIO and then delete the rbd image
#
# NB. this variable definition can be commented out to bypass LUN management
#
# Example:
#
#rbd_devices:
# - { pool: 'rbd', image: 'ansible1', size: '30G', host: 'ceph-1', state: 'present' }
# - { pool: 'rbd', image: 'ansible2', size: '15G', host: 'ceph-1', state: 'present' }
# - { pool: 'rbd', image: 'ansible3', size: '30G', host: 'ceph-1', state: 'present' }
# - { pool: 'rbd', image: 'ansible4', size: '50G', host: 'ceph-1', state: 'present' }
#rbd_devices: {}
# client_connections defines the client ACL's to restrict client access to specific LUNs
# The settings are as follows;
# - image_list is a comma separated list of rbd images of the form <pool name>.<rbd_image_name>
# - chap supplies the user and password the client will use for authentication of the
# form <user>/<password>
# - status shows the intended state of this client definition - 'present' or 'absent'
#
# NB. this definition can be commented out to skip client (nodeACL) management
#
# Example:
#
#client_connections:
# - { client: 'iqn.1994-05.com.redhat:rh7-iscsi-client', image_list: 'rbd.ansible1,rbd.ansible2', chap: 'rh7-iscsi-client/redhat', status: 'present' }
# - { client: 'iqn.1991-05.com.microsoft:w2k12r2', image_list: 'rbd.ansible4', chap: 'w2k12r2/microsoft_w2k12', status: 'absent' }
#client_connections: {}###############
# DEPRECATION #
###############
######################################################
# VARIABLES BELOW SHOULD NOT BE MODIFIED BY THE USER #
# *DO NOT* MODIFY THEM #
######################################################
#container_exec_cmd:
#docker: false
#ceph_volume_debug: "{{ enable_ceph_volume_debug | ternary(1, 0) }}"

Prepare vim group_vars/osds.yml

copy_admin_key: truedevices:
- /dev/sdb
- /dev/sdc

Start Ceph installation

root@openstack-staging:/home/kevin/ceph-ansible# ansible-playbook site.yml  -e container_package_name=docker-ce
INSTALLER STATUS *******************************************************************************************************************************************************
Install Ceph Monitor : Complete (0:01:45)
Install Ceph Manager : Complete (0:05:44)
Install Ceph OSD : Complete (0:01:03)
Install Ceph RGW : Complete (0:00:29)
Install Ceph Client : Complete (0:00:34)
Install Ceph Dashboard : Complete (0:01:07)
Install Ceph Grafana : Complete (0:00:34)
Install Ceph Node Exporter : Complete (0:00:25)
Wednesday 23 June 2021 23:18:27 +0000 (0:00:00.053) 0:14:12.417 ********
===============================================================================
ceph-facts : get ceph current status -------------------------------------------------------------------------------------------------------------------------- 300.25s
ceph-common : install ceph for debian -------------------------------------------------------------------------------------------------------------------------- 87.19s
ceph-mon : waiting for the monitor(s) to form the quorum... ---------------------------------------------------------------------------------------------------- 41.49s
ceph-config : look up for ceph-volume rejected devices --------------------------------------------------------------------------------------------------------- 29.85s
ceph-osd : wait for all osd to be up --------------------------------------------------------------------------------------------------------------------------- 11.42s
ceph-dashboard : create dashboard admin user ------------------------------------------------------------------------------------------------------------------- 11.14s
ceph-osd : use ceph-volume lvm batch to create bluestore osds --------------------------------------------------------------------------------------------------- 9.49s
ceph-mgr : wait for all mgr to be up ---------------------------------------------------------------------------------------------------------------------------- 6.46s
ceph-grafana : wait for grafana to start ------------------------------------------------------------------------------------------------------------------------ 6.30s
ceph-mgr : create ceph mgr keyring(s) on a mon node ------------------------------------------------------------------------------------------------------------- 4.66s
ceph-dashboard : create radosgw system user --------------------------------------------------------------------------------------------------------------------- 4.54s
ceph-mon : fetch ceph initial keys ------------------------------------------------------------------------------------------------------------------------------ 4.34s
ceph-facts : get ceph current status ---------------------------------------------------------------------------------------------------------------------------- 3.87s
ceph-infra : update cache for Debian based OSs ------------------------------------------------------------------------------------------------------------------ 3.83s
ceph-config : create ceph initial directories ------------------------------------------------------------------------------------------------------------------- 3.83s
ceph-config : look up for ceph-volume rejected devices ---------------------------------------------------------------------------------------------------------- 3.77s
ceph-config : create ceph initial directories ------------------------------------------------------------------------------------------------------------------- 3.71s
ceph-dashboard : copy self-signed generated certificate on mons ------------------------------------------------------------------------------------------------- 3.65s
ceph-config : create ceph initial directories ------------------------------------------------------------------------------------------------------------------- 3.40s
ceph-config : create ceph initial directories ------------------------------------------------------------------------------------------------------------------- 3.32s

Ceph Verification

root@openstack-ceph01:/etc/ceph# ceph -s
cluster:
id: a74af1b7-a809-4abf-af89-590d244254f8
health: HEALTH_WARN
mons are allowing insecure global_id reclaim
services:
mon: 3 daemons, quorum openstack-ceph01,openstack-ceph02,openstack-ceph03 (age 51m)
mgr: openstack-ceph02(active, since 40m), standbys: openstack-ceph01, openstack-ceph03
osd: 6 osds: 6 up (since 44m), 6 in (since 44m)
rgw: 3 daemons active (openstack-ceph01.rgw0, openstack-ceph02.rgw0, openstack-ceph03.rgw0)
task status:data:
pools: 9 pools, 233 pgs
objects: 208 objects, 16 MiB
usage: 6.1 GiB used, 762 GiB / 768 GiB avail
pgs: 233 active+clean

Now we’re able to access ceph dashboard and grafana dashboard

Ceph Cluster Initialization

How to Calculate PG number?

We have 48 OSD in this case.

https://access.redhat.com/labs/cephpgc/

Then we can download the commands

## Note:
## The 'while' loops below pause between pools to allow all PGs to be created.
## This is a safety mechanism to prevent saturating the Monitor nodes.
##-------------------------------------------------------------------
ceph osd pool create backups 64
ceph osd pool set backups size 3
while [ $(ceph -s | grep creating -c) -gt 0 ]; do echo -n .;sleep 1; done
ceph osd pool create volumes 1024
ceph osd pool set volumes size 3
while [ $(ceph -s | grep creating -c) -gt 0 ]; do echo -n .;sleep 1; done
ceph osd pool create vms 512
ceph osd pool set vms size 3
while [ $(ceph -s | grep creating -c) -gt 0 ]; do echo -n .;sleep 1; done
ceph osd pool create images 128
ceph osd pool set images size 3
while [ $(ceph -s | grep creating -c) -gt 0 ]; do echo -n .;sleep 1; done

Init the pool

rbd pool init volumes
rbd pool init images
rbd pool init backups
rbd pool init vms

Create keystring

root@openstack-ceph01:/home/kevin# docker exec -it ceph-mgr-openstack-ceph01 bash
ceph auth get-or-create client.glance mon 'profile rbd' osd 'profile rbd pool=images' mgr 'profile rbd pool=images' -o /etc/ceph/ceph.client.glance.keyring
ceph auth get-or-create client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images' mgr 'profile rbd pool=volumes, profile rbd pool=vms' -o /etc/ceph/ceph.client.cinder.keyring
ceph auth get-or-create client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups' mgr 'profile rbd pool=backups' -o /etc/ceph/ceph.client.cinder-backup.keyring

And then we need copy all these 3 files to staging machine /etc/kolla/config foler.

root@ems-la4-staging:/etc/kolla# mkdir  -p /etc/kolla/config/glance/
root@ems-la4-staging:/etc/kolla# mkdir -p /etc/kolla/config/cinder/cinder-backup/
root@ems-la4-staging:/etc/kolla# mkdir -p /etc/kolla/config/cinder/cinder-volume/
root@ems-la4-staging:/etc/kolla# mkdir -p /etc/kolla/config/nova/

Copy file to staging machine

scp /etc/ceph/ceph.client.glance.keyring root@<staging-machine>:/etc/kolla/config/glance/
scp /etc/ceph/ceph.client.cinder.keyring root@<staging-machine>:/etc/kolla/config/cinder/cinder-backup/
scp /etc/ceph/ceph.client.cinder.keyring root@<staging-machine>:/etc/kolla/config/cinder/cinder-volume/
scp /etc/ceph/ceph.client.cinder-backup.keyring root@<staging-machine>:/etc/kolla/config/cinder/cinder-backup/
scp /etc/ceph/ceph.client.cinder.keyring root@<staging-machine>:/etc/kolla/config/nova/

Now, we have ceph cluster running, and in next session, I will add some troubleshooting guide you may experience during ceph cluster installation.

--

--

Network Engineer

Love podcasts or audiobooks? Learn on the go with our new app.

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store