#!/usr/bin/python

"""

The script "gfs2_lockcapture" will capture locking information from GFS2 file

systems and DLM.

@author    : Shane Bradley

@contact   : sbradley@redhat.com

@version   : 0.95

@copyright : GPLv2

"""

import sys

import os

import os.path

import logging

import logging.handlers

from optparse import OptionParser, Option, SUPPRESS_HELP

import time

import platform

import shutil

import subprocess

import tarfile

# #####################################################################

# Global vars:

# #####################################################################

"""

@cvar VERSION_NUMBER: The version number of this script.

@type VERSION_NUMBER: String

@cvar MAIN_LOGGER_NAME: The name of the logger.

@type MAIN_LOGGER_NAME: String

@cvar PATH_TO_DEBUG_DIR: The path to the debug directory for the linux kernel.

@type PATH_TO_DEBUG_DIR: String

@cvar PATH_TO_PID_FILENAME: The path to the pid file that will be used to make

sure only 1 instance of this script is running at any time.

@type PATH_TO_PID_FILENAME: String

"""

VERSION_NUMBER = "0.9-8"

MAIN_LOGGER_NAME = "%s" %(os.path.basename(sys.argv[0]))

PATH_TO_DEBUG_DIR="/sys/kernel/debug"

PATH_TO_PID_FILENAME = "/var/run/%s.pid" %(os.path.basename(sys.argv[0]))

# #####################################################################

# Class to define what a clusternode is.

# #####################################################################

class ClusterNode:

    """

    This class represents a cluster node that is a current member in a cluster.

    """

    def __init__(self, clusternodeName, clusternodeID, clusterName, mapOfMountedFilesystemLabels):

        """

        @param clusternodeName: The name of the cluster node.

        @type clusternodeName: String

        @param clusterName: The name of the cluster that this cluster node is a

        member of.

        @param clusternodeID: The id of the cluster node.

        @type clusternodeID: Int

        @param clusterName: The name of the cluster that this cluster node is a

        @type clusterName: String

        @param mapOfMountedFilesystemLabels: A map of filesystem labels(key) for

        a mounted filesystem. The value is the line for the matching mounted

        filesystem from the mount -l command.

        @type mapOfMountedFilesystemLabels: Dict

        """

        self.__clusternodeName = clusternodeName

        self.__clusternodeID  = clusternodeID

        self.__clusterName = clusterName

        self.__mapOfMountedFilesystemLabels = mapOfMountedFilesystemLabels

    def __str__(self):

        """

        This function will return a string representation of the object.

        @return: Returns a string representation of the object.

        @rtype: String

        """

        rString = ""

        rString += "%s:%s(id:%d)" %(self.getClusterName(), self.getClusterNodeName(), self.getClusterNodeID())

        fsLabels = list(self.__mapOfMountedFilesystemLabels.keys())

        fsLabels.sort()

        for fsLabel in fsLabels:

            rString += "\n\t%s --> %s" %(fsLabel, self.__mapOfMountedFilesystemLabels.get(fsLabel))

        return rString.rstrip()

    def getClusterNodeName(self):

        """

        Returns the name of the cluster node.

        @return: Returns the name of the cluster node.

        @rtype: String

        """

        return self.__clusternodeName

    def getClusterNodeID(self):

        """

        Returns the id of the cluster node.

        @return: Returns the id of the cluster node.

        @rtype: String

        """

        return self.__clusternodeID

    def getClusterName(self):

        """

        Returns the name of cluster that this cluster node is a member of.

        @return: Returns the name of cluster that this cluster node is a member

        of.

        @rtype: String

        """

        return self.__clusterName

    def getMountedGFS2FilesystemNames(self, includeClusterName=True):

        """

        Returns the names of all the mounted GFS2 filesystems. By default

        includeClusterName is True which will include the name of the cluster

        and the GFS2 filesystem name(ex. f18cluster:mygfs2vol1) in the list of

        mounted GFS2 filesystems. If includeClusterName is False it will only

        return a list of all the mounted GFS2 filesystem names(ex. mygfs2vol1).

        @return: Returns a list of all the mounted GFS2 filesystem names.

        @rtype: Array

        @param includeClusterName: By default this option is True and will

        include the name of the cluster and the GFS2 filesystem name. If False

        then only the GFS2 filesystem name will be included.

        @param includeClusterName: Boolean

        """

        # If true will prepend the cluster name to gfs2 fs name

        if (includeClusterName):

            return list(self.__mapOfMountedFilesystemLabels.keys())

        else:

            listOfGFS2MountedFilesystemLabels = []

            for fsLabel in list(self.__mapOfMountedFilesystemLabels.keys()):

                fsLabelSplit = fsLabel.split(":", 1)

                if (len(fsLabelSplit) == 2):

                    listOfGFS2MountedFilesystemLabels.append(fsLabelSplit[1])

            return listOfGFS2MountedFilesystemLabels

    def getMountedGFS2FilesystemPaths(self):

        """

        Returns a map of all the mounted GFS2 filesystem paths. The key is the

        GFS2 fs name(clustername:fs name) and value is the mountpoint.

        @return: Returns a map of all the mounted GFS2 filesystem paths. The key

        is the GFS2 fs name(clustername:fs name) and value is the mountpoint.

        Returns a list of all the mounted GFS2 filesystem paths.

        @rtype: Map

        """

        mapOfGFS2MountedFilesystemPaths = {}

        for fsLabel in list(self.__mapOfMountedFilesystemLabels.keys()):

            value = self.__mapOfMountedFilesystemLabels.get(fsLabel)

            mountPoint = value.split("type", 1)[0].split("on")[1]

            if (len(mountPoint) > 0):

                mapOfGFS2MountedFilesystemPaths[fsLabel] = mountPoint

        return mapOfGFS2MountedFilesystemPaths

# #####################################################################

# Helper functions.

# #####################################################################

def runCommand(command, listOfCommandOptions, standardOut=subprocess.PIPE, standardError=subprocess.PIPE):

    """

    This function will execute a command. It will return True if the return code

    was zero, otherwise False is returned.

    @return: Returns True if the return code was zero, otherwise False is

    returned.

    @rtype: Boolean

    @param command: The command that will be executed.

    @type command: String

    @param listOfCommandOptions: The list of options for the command that will

    be executed.

    @type listOfCommandOptions: Array

    @param standardOut: The pipe that will be used to write standard output. By

    default the pipe that is used is subprocess.PIPE.

    @type standardOut: Pipe

    @param standardError: The pipe that will be used to write standard error. By

    default the pipe that is used is subprocess.PIPE.

    @type standardError: Pipe

    """

    stdout = ""

    stderr = ""

    try:

        commandList = [command]

        commandList += listOfCommandOptions

        task = subprocess.Popen(commandList, stdout=standardOut, stderr=standardError)

        task.wait()

        (stdout, stderr) = task.communicate()

        return (task.returncode == 0)

    except OSError:

        commandOptionString = ""

        for option in listOfCommandOptions:

            commandOptionString += "%s " %(option)

        message = "An error occurred running the command: $ %s %s" %(command, commandOptionString)

        if (len(stdout.rstrip()) > 0):

            message += "\n%s" %(stdout.rstrip())

        if (len(stderr.rstrip()) > 0):

            message += "\n%s" %(stderr.rstrip())

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

    return False

def runCommandOutput(command, listOfCommandOptions, standardOut=subprocess.PIPE, standardError=subprocess.PIPE):

    """

    This function will execute a command. Returns the output that was written to standard output. None is

    returned if there was an error.

    @return: Returns the output that was written to standard output. None is

    returned if there was an error.

    @rtype: String

    @param command: The command that will be executed.

    @type command: String

    @param listOfCommandOptions: The list of options for the command that will

    be executed.

    @type listOfCommandOptions: Array

    @param standardOut: The pipe that will be used to write standard output. By

    default the pipe that is used is subprocess.PIPE.

    @type standardOut: Pipe

    @param standardError: The pipe that will be used to write standard error. By

    default the pipe that is used is subprocess.PIPE.

    @type standardError: Pipe

    """

    stdout = ""

    stderr = ""

    try:

        commandList = [command]

        commandList += listOfCommandOptions

        task = subprocess.Popen(commandList, stdout=standardOut, stderr=standardError)

        task.wait()

        (stdout, stderr) = task.communicate()

    except OSError:

        commandOptionString = ""

        for option in listOfCommandOptions:

            commandOptionString += "%s " %(option)

        message = "An error occurred running the command: $ %s %s" %(command, commandOptionString)

        if (len(stdout.rstrip()) > 0):

            message += "\n%s" %(stdout.rstrip())

        if (len(stderr.rstrip()) > 0):

            message += "\n%s" %(stderr.rstrip())

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

        return None

    return stdout.decode().strip().rstrip()

def writeToFile(pathToFilename, data, appendToFile=True, createFile=False):

    """

    This function will write a string to a file.

    @return: Returns True if the string was successfully written to the file,

    otherwise False is returned.

    @rtype: Boolean

    @param pathToFilename: The path to the file that will have a string written

    to it.

    @type pathToFilename: String

    @param data: The string that will be written to the file.

    @type data: String

    @param appendToFile: If True then the data will be appened to the file, if

    False then the data will overwrite the contents of the file.

    @type appendToFile: Boolean

    @param createFile: If True then the file will be created if it does not

    exists, if False then file will not be created if it does not exist

    resulting in no data being written to the file.

    @type createFile: Boolean

    """

    [parentDir, filename] = os.path.split(pathToFilename)

    if (os.path.isfile(pathToFilename) or (os.path.isdir(parentDir) and createFile)):

        try:

            filemode = "w"

            if (appendToFile):

                filemode = "a"

            fout = open(pathToFilename, filemode)

            fout.write(data + "\n")

            fout.close()

            return True

        except UnicodeEncodeError as e:

            message = "There was a unicode encode error writing to the file: %s." %(pathToFilename)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

        except IOError:

            message = "There was an error writing to the file: %s." %(pathToFilename)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

    return False

def mkdirs(pathToDSTDir):

    """

    This function will attempt to create a directory with the path of the value of pathToDSTDir.

    @return: Returns True if the directory was created or already exists.

    @rtype: Boolean

    @param pathToDSTDir: The path to the directory that will be created.

    @type pathToDSTDir: String

    """

    if (os.path.isdir(pathToDSTDir)):

        return True

    elif ((not os.access(pathToDSTDir, os.F_OK)) and (len(pathToDSTDir) > 0)):

        try:

            os.makedirs(pathToDSTDir)

        except (OSError, os.error):

            message = "Could not create the directory: %s." %(pathToDSTDir)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

        except (IOError, os.error):

            message = "Could not create the directory with the path: %s." %(pathToDSTDir)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

    return os.path.isdir(pathToDSTDir)

def removePIDFile():

    """

    This function will remove the pid file.

    @return: Returns True if the file was successfully remove or does not exist,

    otherwise False is returned.

    @rtype: Boolean

    """

    message = "Removing the pid file: %s" %(PATH_TO_PID_FILENAME)

    logging.getLogger(MAIN_LOGGER_NAME).debug(message)

    if (os.path.exists(PATH_TO_PID_FILENAME)):

        try:

            os.remove(PATH_TO_PID_FILENAME)

        except IOError:

            message = "There was an error removing the file: %s." %(PATH_TO_PID_FILENAME)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

    return os.path.exists(PATH_TO_PID_FILENAME)

def archiveData(pathToSrcDir):

    """

    This function will return the path to the tar.bz2 file that was created. If

    the tar.bz2 file failed to be created then an empty string will be returned

    which would indicate an error occurred.

    @return: This function will return the path to the tar.bz2 file that was

    created. If the tar.bz2 file failed to be created then an empty string will

    be returned which would indicate an error occurred.

    @rtype: String

    @param pathToSrcDir: The path to the directory that will be archived into a

    .tar.bz2 file.

    @type pathToSrcDir: String

    """

    if (os.path.exists(pathToSrcDir)):

        pathToTarFilename = "%s-%s.tar.bz2" %(pathToSrcDir, platform.node())

        if (os.path.exists(pathToTarFilename)):

            message = "A compressed archvied file already exists and will be removed: %s" %(pathToTarFilename)

            logging.getLogger(MAIN_LOGGER_NAME).status(message)

            try:

                os.remove(pathToTarFilename)

            except IOError:

                message = "There was an error removing the file: %s." %(pathToTarFilename)

                logging.getLogger(MAIN_LOGGER_NAME).error(message)

                return ""

        message = "Creating a compressed archvied file: %s" %(pathToTarFilename)

        logging.getLogger(MAIN_LOGGER_NAME).status(message)

        try:

            tar = tarfile.open(pathToTarFilename, "w:bz2")

            tar.add(pathToSrcDir, arcname=os.path.basename(pathToSrcDir))

            tar.close()

        except tarfile.TarError:

            message = "There was an error creating the tarfile: %s." %(pathToTarFilename)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return ""

        if (os.path.exists(pathToTarFilename)):

            return pathToTarFilename

    return ""

def getDataFromFile(pathToSrcFile) :

    """

    This function will return the data in an array. Where each newline in file

    is a seperate item in the array. This should really just be used on

    relatively small files.

    None is returned if no file is found.

    @return: Returns an array of Strings, where each newline in file is an item

    in the array.

    @rtype: Array

    @param pathToSrcFile: The path to the file which will be read.

    @type pathToSrcFile: String

    """

    if (len(pathToSrcFile) > 0) :

        try:

            fin = open(pathToSrcFile, "r")

            data = fin.readlines()

            fin.close()

            return data

        except (IOError, os.error):

            message = "An error occured reading the file: %s." %(pathToSrcFile)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

    return None

def copyFile(pathToSrcFile, pathToDstFile):

    """

    This function will copy a src file to dst file.

    @return: Returns True if the file was copied successfully.

    @rtype: Boolean

    @param pathToSrcFile: The path to the source file that will be copied.

    @type pathToSrcFile: String

    @param pathToDstFile: The path to the destination of the file.

    @type pathToDstFile: String

    """

    if(not os.path.exists(pathToSrcFile)):

        message = "The file does not exist with the path: %s." %(pathToSrcFile)

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

        return False

    elif (not os.path.isfile(pathToSrcFile)):

        message = "The path to the source file is not a regular file: %s." %(pathToSrcFile)

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

        return False

    elif (pathToSrcFile == pathToDstFile):

        message = "The path to the source file and path to destination file cannot be the same: %s." %(pathToDstFile)

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

        return False

    else:

        # Create the directory structure if it does not exist.

        (head, tail) = os.path.split(pathToDstFile)

        if (not mkdirs(head)) :

            # The path to the directory was not created so file

            # could not be copied.

            return False

        # Copy the file to the dst path.

        try:

            shutil.copy(pathToSrcFile, pathToDstFile)

        except shutil.Error:

            message = "Cannot copy the file %s to %s." %(pathToSrcFile, pathToDstFile)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

        except OSError:

            message = "Cannot copy the file %s to %s." %(pathToSrcFile, pathToDstFile)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

        except IOError:

            message = "Cannot copy the file %s to %s." %(pathToSrcFile, pathToDstFile)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

        return (os.path.exists(pathToDstFile))

def copyDirectory(pathToSrcDir, pathToDstDir):

    """

    This function will copy a src dir to dst dir.

    @return: Returns True if the dir was copied successfully.

    @rtype: Boolean

    @param pathToSrcDir: The path to the source dir that will be copied.

    @type pathToSrcDir: String

    @param pathToDstDir: The path to the destination of the dir.

    @type pathToDstDir: String

    """

    if(not os.path.exists(pathToSrcDir)):

        message = "The directory does not exist with the path: %s." %(pathToSrcDir)

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

        return False

    elif (not os.path.isdir(pathToSrcDir)):

        message = "The path to the source directory is not a directory: %s." %(pathToSrcDir)

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

        return False

    elif (pathToSrcDir == pathToDstDir):

        message = "The path to the source directory and path to destination directory cannot be the same: %s." %(pathToDstDir)

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

        return False

    else:

        if (not mkdirs(pathToDstDir)) :

            # The path to the directory was not created so file

            # could not be copied.

            return False

        # Copy the file to the dst path.

        dst = os.path.join(pathToDstDir, os.path.basename(pathToSrcDir))

        try:

            shutil.copytree(pathToSrcDir, dst)

        except shutil.Error:

            message = "Cannot copy the directory %s to %s." %(pathToSrcDir, dst)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

        except OSError:

            message = "Cannot copy the directory %s to %s." %(pathToSrcDir, dst)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

        except IOError:

            message = "Cannot copy the directory %s to %s." %(pathToSrcDir, dst)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

            return False

        return (os.path.exists(dst))

def backupOutputDirectory(pathToOutputDir):

    """

    This function will return True if the pathToOutputDir does not exist or the

    directory was successfully rename. If pathToOutputDir exists and was not

    successfully rename then False is returned.

    @return: Returns True if the pathToOutputDir does not exist or the directory

    was successfully rename. If pathToOutputDir exists and was not successfully

    rename then False is returned.

    @rtype: Boolean

    @param pathToOutputDir: The path to the directory that will be backed up.

    @type pathToOutputDir: String

    """

    if (os.path.exists(pathToOutputDir)):

        message = "The path already exists and could contain previous lockdump data: %s" %(pathToOutputDir)

        logging.getLogger(MAIN_LOGGER_NAME).info(message)

        backupIndex = 1

        pathToDST = ""

        keepSearchingForIndex = True

        while (keepSearchingForIndex):

            pathToDST = "%s.bk-%d" %(pathToOutputDir, backupIndex)

            if (os.path.exists(pathToDST)):

                backupIndex += 1

            else:

                keepSearchingForIndex = False

        try:

            message = "The existing output directory will be renamed: %s to %s." %(pathToOutputDir, pathToDST)

            logging.getLogger(MAIN_LOGGER_NAME).status(message)

            shutil.move(pathToOutputDir, pathToDST)

        except shutil.Error:

            message = "There was an error renaming the directory: %s to %s." %(pathToOutputDir, pathToDST)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

        except OSError:

            message = "There was an error renaming the directory: %s to %s." %(pathToOutputDir, pathToDST)

            logging.getLogger(MAIN_LOGGER_NAME).error(message)

    # The path should not exists now, else there was an error backing up an

    # existing output directory.

    return (not os.path.exists(pathToOutputDir))

def mountFilesystem(filesystemType, pathToDevice, pathToMountPoint):

    """

    This function will attempt to mount a filesystem. If the filesystem is

    already mounted or the filesystem was successfully mounted then True is

    returned, otherwise False is returned.

    @return: If the filesystem is already mounted or the filesystem was

    successfully mounted then True is returned, otherwise False is returned.

    @rtype: Boolean

    @param filesystemType: The type of filesystem that will be mounted.

    @type filesystemType: String

    @param pathToDevice: The path to the device that will be mounted.

    @type pathToDevice: String

    @param pathToMountPoint: The path to the directory that will be used as the

    mount point for the device.

    @type pathToMountPoint: String

    """

    if (os.path.ismount(PATH_TO_DEBUG_DIR)):

        return True

    listOfCommandOptions = ["-t", filesystemType, pathToDevice, pathToMountPoint]

    if (not runCommand("mount", listOfCommandOptions)):

        message = "There was an error mounting the filesystem type %s for the device %s to the mount point %s." %(filesystemType, pathToDevice, pathToMountPoint)

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

    return  os.path.ismount(PATH_TO_DEBUG_DIR)

def exitScript(removePidFile=True, errorCode=0):

    """

    This function will cause the script to exit or quit. It will return an error

    code and will remove the pid file that was created.

    @param removePidFile: If True(default) then the pid file will be remove

    before the script exits.

    @type removePidFile: Boolean

    @param errorCode: The exit code that will be returned. The default value is 0.

    @type errorCode: Int

    """

    if (removePidFile):

        removePIDFile()

    message = "The script will exit."

    logging.getLogger(MAIN_LOGGER_NAME).info(message)

    sys.exit(errorCode)

# #####################################################################

# Helper functions for gathering the lockdumps.

# #####################################################################

def getClusterNode(listOfGFS2Names):

    """

    This function return a ClusterNode object if the machine is a member of a

    cluster and has GFS2 filesystems mounted for that cluster. The

    listOfGFS2Names is a list of GFS2 filesystem that need to have their data

    capture. If the list is empty then that means that all the mounted GFS2

    filesystems will be captured, if list is not empty then only those GFS2

    filesystems in the list will have their data captured.

    @return: Returns a cluster node object if there was mounted GFS2 filesystems

    found that will have their data captured.

    @rtype: ClusterNode

    @param listOfGFS2Names: A list of GFS2 filesystem names that will have their

    data captured.  If the list is empty then that means that all the mounted

    GFS2 filesystems will be captured, if list is not empty then only those GFS2

    filesystems in the list will have their data captured.

    @type listOfGFS2Names: Array

    """

    # Return a ClusterNode object if the clusternode and cluster name are found

    # in the output, else return None.

    clusterName = ""

    clusternodeName = ""

    clusternodeID = ""

    if (runCommand("which", ["cman_tool"])):

        stdout = runCommandOutput("cman_tool", ["status"])

        if (not stdout == None):

            stdoutSplit = stdout.split("\n")

            clusterName = ""

            clusternodeName = ""

            for line in stdoutSplit:

                if (line.startswith("Cluster Name:")):

                    clusterName = line.split("Cluster Name:")[1].strip().rstrip()

                if (line.startswith("Node name: ")):

                    clusternodeName = line.split("Node name:")[1].strip().rstrip()

                if (line.startswith("Node ID: ")):

                    clusternodeID = line.split("Node ID: ")[1].strip().rstrip()

    elif (runCommand("which", ["corosync-cmapctl"])):

        # Another way to get the local cluster node is: $ crm_node -i; crm_node -l

        # Get the name of the cluster.

        stdout = runCommandOutput("corosync-cmapctl", ["-g", "totem.cluster_name"])

        if (not stdout == None):

            stdoutSplit = stdout.split("=")

            if (len(stdoutSplit) == 2):

                clusterName = stdoutSplit[1].strip().rstrip()

        # Get the id of the local cluster node so we can get the clusternode name

        clusternodeID = ""

        stdout = runCommandOutput("corosync-cmapctl", ["-g", "runtime.votequorum.this_node_id"])

        if (not stdout == None):

            stdoutSplit = stdout.split("=")

            if (len(stdoutSplit) == 2):

               clusternodeID = stdoutSplit[1].strip().rstrip()

        # Now that we the nodeid then we can get the clusternode name.

        if (len(clusternodeID) > 0):

            stdout = runCommandOutput("corosync-quorumtool", ["-l"])

            if (not stdout == None):

                for line in stdout.split("\n"):

                    if (line.find("local") >=0):

                        splitLine = line.split(" (local)")

                        clusternodeName = splitLine[0].split()[2]

                        break;

    # If a clusternode name and cluster name was found then return a new object

    # since this means this cluster is part of cluster.

    if ((len(clusterName) > 0) and (len(clusternodeName) > 0)):

        mapOfMountedFilesystemLabels = getLabelMapForMountedFilesystems(clusterName, getMountedGFS2Filesystems())

        # These will be the GFS2 filesystems that will have their lockdump information gathered.

        if (len(listOfGFS2Names) > 0):

            for label in list(mapOfMountedFilesystemLabels.keys()):

                foundMatch = False

                for gfs2FSName in listOfGFS2Names:

                    if ((gfs2FSName == label) or ("%s:%s"%(clusterName, gfs2FSName) == label)):

                        foundMatch = True

                        break

                if ((not foundMatch) and (label in mapOfMountedFilesystemLabels)):

                    del(mapOfMountedFilesystemLabels[label])

        # Cast the node id to an int, and default is 0 if node is not found or

        # not castable.

        clusternodeIDInt = 0

        if (clusternodeID.isalnum()):

            try:

                clusternodeIDInt = int(clusternodeID)

            except(ValueError):

                pass

        return ClusterNode(clusternodeName, clusternodeIDInt, clusterName, mapOfMountedFilesystemLabels)

    else:

        return None

def parse_dlm_ls(dlm_ls):

    """

    This function returns the names of all the dlm lockspace names found with the

    commands "dlm_tool ls" or "group_tool ls" output.

    @return: A list of all the dlm lockspace names.

    @rtype: Array

    """

    dlmLockspaces = []

    if (not dlm_ls == None):

        dlm_ls = dlm_ls.replace("dlm lockspaces\n", "")

        dlmToolLSKeys = ["name", "id", "flags", "change", "members"]

        # Split on newlines

        dlm_lsSections = dlm_ls.split("\n\n")

        for section in dlm_lsSections:

            # Create tmp map to hold data

            if (section.startswith("fence domain")):

                # Not concerned with fence information.

                continue

            dlmToolLSMap = dict.fromkeys(dlmToolLSKeys)

            lines = section.split("\n")

            for line in lines:

                for dlmToolLSKey in list(dlmToolLSMap.keys()):

                    if (line.startswith(dlmToolLSKey)):

                        value = line.replace(dlmToolLSKey, " ", 1).strip().rstrip()

                        dlmToolLSMap[dlmToolLSKey] = value

                if ((not dlmToolLSMap.get("name") == None) and (not dlmToolLSMap.get("id") == None)):

                    dlmLockspaces.append(dlmToolLSMap.get("name"))

    return dlmLockspaces

def getGroupToolDLMLockspaces():

    """

    This function returns the names of all the dlm lockspace names found with the

    command: "group_tool ls".

    @return: A list of all the dlm lockspace names.

    @rtype: Array

    """

    dlmLockspaces = []

    stdout = runCommandOutput("group_tool", ["ls"])

    if (not stdout == None):

        lines = stdout.split("\n")

        if (len(lines) > 0):

            if (lines[0].startswith("type")):

                # Then running cman-2.0

                for line in lines:

                    if (line.startswith("dlm")):

                        dlmLockspaces.append(line.split()[2])

            else:

                # Then running cman-3.0 and uses same sorta output as `dlm_tool ls`.

                dlmLockspaces = parse_dlm_ls(stdout)

    return dlmLockspaces

def getDLMLockspaces():

    """

    Returns a list of the dlm lockspace names.

    @return: Returns a list of dlm lockspace names.

    @rtype: Array

    """

    message = "Gathering the DLM Lockspace Names."

    logging.getLogger(MAIN_LOGGER_NAME).debug(message)

    dlmLockspaces = parse_dlm_ls(runCommandOutput("dlm_tool", ["ls"]))

    if (not len(dlmLockspaces) > 0):

        message = "There was no dlm lockspaces found with the \"dlm_tool ls\" command.  Trying with the \"group_tool ls\" command."

        logging.getLogger(MAIN_LOGGER_NAME).debug(message)

        dlmLockspaces = getGroupToolDLMLockspaces()

    return dlmLockspaces

def getVerifiedDLMLockspaceNames(lockspaceNames):

    """

    Returns a list of DLM lockspaces that have been verified to exists in the

    command output of $(dlm_tool ls).

    @return: Returns a list of DLM lockspaces that have been verified to exists

    in the command output of $(dlm_tool ls).

    @rtype: Array

    @param lockspaceNames: This is the list of DLM lockspaces that will have

    their debug directory copied.

    @type lockspaceNames: Array

    """

    # Get a list of all the DLM lockspaces names.

    dlmLockspaces = getDLMLockspaces()

    # Verify the lockspaceNames are lockspaces that exist.

    verifiedLockspaceNames = []

    for lockspaceName in lockspaceNames:

        if ((lockspaceName in dlmLockspaces) and

            (not lockspaceName in verifiedLockspaceNames)):

            verifiedLockspaceNames.append(lockspaceName)

    return verifiedLockspaceNames

def getMountedGFS2Filesystems():

    """

    This function returns a list of all the mounted GFS2 filesystems.

    @return: Returns a list of all the mounted GFS2 filesystems.

    @rtype: Array

    """

    fsType = "gfs2"

    listOfMountedFilesystems = []

    stdout = runCommandOutput("mount", ["-l"])

    if (not stdout == None):

        stdoutSplit = stdout.split("\n")

        for line in stdoutSplit:

            splitLine = line.split()

            if (len(splitLine) >= 5):

                if (splitLine[4] == fsType):

                    listOfMountedFilesystems.append(line)

    return listOfMountedFilesystems

def getLabelMapForMountedFilesystems(clusterName, listOfMountedFilesystems):

    """

    This function will return a dictionary of the mounted GFS2 filesystem that

    contain a label that starts with the cluster name. For example:

    {'f18cluster:mygfs2vol1': '/dev/vdb1 on /mnt/gfs2vol1 type gfs2 (rw,relatime) [f18cluster:mygfs2vol1]'}

    @return: Returns a dictionary of the mounted GFS2 filesystems that contain a

    label that starts with the cluster name.

    @rtype: Dict

    @param clusterName: The name of the cluster.

    @type clusterName: String

    @param listOfMountedFilesystems: A list of all the mounted GFS2 filesystems.

    @type listOfMountedFilesystems: Array

    """

    mapOfMountedFilesystemLabels = {}

    for mountedFilesystem in listOfMountedFilesystems:

        splitMountedFilesystem = mountedFilesystem.split()

        fsLabel = splitMountedFilesystem[-1].strip().strip("[").rstrip("]")

        if (len(fsLabel) > 0):

            # Verify it starts with name of the cluster.

            if (fsLabel.startswith("%s:" %(clusterName))):

                mapOfMountedFilesystemLabels[fsLabel] = mountedFilesystem

    return mapOfMountedFilesystemLabels

# #####################################################################

# Gather output from command functions

# #####################################################################

def gatherHostData(pathToDSTDir):

    """

    This function will gather general information about the cluster and write

    the results to a file. The following data will be captured: hostname, date,

    uname -a, uptime.

    @param pathToDSTDir: This is the path to directory where the files will be

    written to.

    @type pathToDSTDir: String

    """

    # Gather some general information and write to system.txt.

    systemString = "HOSTNAME=%s\nTIMESTAMP=%s\n" %(platform.node(), time.strftime("%Y-%m-%d %H:%M:%S"))

    stdout = runCommandOutput("uname", ["-a"]).strip().rstrip()

    if (not stdout == None):

        systemString += "UNAMEA=%s\n" %(stdout)

    stdout = runCommandOutput("uptime", []).strip().rstrip()

    if (not stdout == None):

        systemString += "UPTIME=%s" %(stdout)

    writeToFile(os.path.join(pathToDSTDir, "hostinformation.txt"), systemString, createFile=True)

def gatherDiagnosticData(pathToDSTDir):

    """

    This function will gather general information about the cluster and write (or

    copy) the results to a file.

    @param pathToDSTDir: This is the path to directory where the files will be

    written to.

    @type pathToDSTDir: String

    """

    # Get "ps -eo user,pid,%cpu,%mem,vsz,rss,tty,stat,start,time,comm,wchan" data.

    # Get " ps h -AL -o tid,s,cmd

    command = "ps"

    pathToCommandOutput = os.path.join(pathToDSTDir, "ps_hALo-tid.s.cmd")

    try:

        fout = open(pathToCommandOutput, "w")

        #runCommand(command, ["-eo", "user,pid,%cpu,%mem,vsz,rss,tty,stat,start,time,comm,wchan"], standardOut=fout)

        runCommand(command, ["h", "-AL", "-o", "tid,s,cmd"], standardOut=fout)

        fout.close()

    except IOError:

        message = "There was an error writing the command output for %s to the file %s." %(command, pathToCommandOutput)

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

    # Copy misc files

    pathToSrcFile = "/proc/mounts"

    copyFile(pathToSrcFile, os.path.join(pathToDSTDir, pathToSrcFile.strip("/")))

    pathToSrcFile = "/proc/slabinfo"

    copyFile(pathToSrcFile, os.path.join(pathToDSTDir, pathToSrcFile.strip("/")))

    # Copy the DLM hash table sizes:

    pathToHashTableFiles = ["/sys/kernel/config/dlm/cluster/lkbtbl_size", "/sys/kernel/config/dlm/cluster/dirtbl_size",

                            "/sys/kernel/config/dlm/cluster/rsbtbl_size"]

    for pathToSrcFile in pathToHashTableFiles:

        if (os.path.exists(pathToSrcFile)):

            copyFile(pathToSrcFile, os.path.join(pathToDSTDir, pathToSrcFile.strip("/")))

def gatherOptionalDiagnosticData(pathToDSTDir):

    """

    This function will gather optional information about the cluster and write

    the results to a file.

    @param pathToDSTDir: This is the path to directory where the files will be

    written to.

    @type pathToDSTDir: String

    """

    # Get df -h ouput

    command = "df"

    pathToCommandOutput = os.path.join(pathToDSTDir, "df-h.cmd")

    try:

        fout = open(pathToCommandOutput, "w")

        runCommand(command, ["-h"], standardOut=fout)

        fout.close()

    except IOError:

        message = "There was an error writing the command output for %s to the file %s." %(command, pathToCommandOutput)

        logging.getLogger(MAIN_LOGGER_NAME).error(message)

    # Write the status of all the nodes in the cluster out.

    if (runCommand("which", ["cman_tool"])):

        command = "cman_tool"

        pathToCommandOutput = os.path.join(pathToDSTDir, "cman_tool_status")

        try: