python-fabio 0.3.0+dfsg-1build1 / usr / lib / python2.7 / dist-packages / fabio / cbfimage.py

# coding: utf-8
#
#    Project: X-ray image reader
#             https://github.com/kif/fabio
#
#
#    Copyright (C) European Synchrotron Radiation Facility, Grenoble, France
#
#    Principal author:       Jérôme Kieffer (Jerome.Kieffer@ESRF.eu)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

"""
Authors: Jérôme Kieffer, ESRF
         email:jerome.kieffer@esrf.fr

Cif Binary Files images are 2D images written by the Pilatus detector and others.
They use a modified (simplified) byte-offset algorithm.

CIF is a library for manipulating Crystallographic information files and tries
to conform to the specification of the IUCR
"""

# get ready for python3
from __future__ import with_statement, print_function, absolute_import

__author__ = "Jérôme Kieffer"
__contact__ = "jerome.kieffer@esrf.eu"
__license__ = "GPLv3+"
__date__ = "30/10/2015"
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
__version__ = ["Generated by CIF.py: Jan 2005 - Oct 2015",
              "Written by Jerome Kieffer: Jerome.Kieffer@esrf.eu",
              "On-line data analysis / ISDD ", "ESRF Grenoble (France)"]


import os, logging
import numpy
from .fabioimage import FabioImage
from .fabioutils import six
from .compression import compByteOffset, decByteOffset, md5sum

logger = logging.getLogger("CbfImage")

# import time
try:
    from ._cif import split_tokens
except ImportError:
    split_tokens = None

DATA_TYPES = { "signed 8-bit integer"   : "int8",
               "signed 16-bit integer"  : "int16",
               "signed 32-bit integer"  : "int32",
               "signed 64-bit integer"  : "int64",
               "unsigned 8-bit integer" : "uint8",
               "unsigned 16-bit integer": "uint16",
               "unsigned 32-bit integer": "uint32",
               "unsigned 64-bit integer": "uint64"
                }

MINIMUM_KEYS = ["X-Binary-Size-Fastest-Dimension",
                "X-Binary-Size-Second-Dimension",
                "X-Binary-Size",
                "X-Binary-Number-of-Elements",
                'X-Binary-Element-Type',
                'X-Binary-Number-of-Elements']


STARTER = b"\x0c\x1a\x04\xd5"
PADDING = 512
BINARAY_SECTION = b"--CIF-BINARY-FORMAT-SECTION--"
CIF_BINARY_BLOCK_KEY = "_array_data.data"

class CbfImage(FabioImage):
    """
    Read the Cif Binary File data format
    """
    def __init__(self, data=None , header=None, fname=None):
        """
        Constructor of the class CIF Binary File reader.

        @param _strFilename: the name of the file to open
        @type  _strFilename: string
        """
        FabioImage.__init__(self, data, header)
        self.cif = CIF()
        self.cbs = None
        self.start_binary = None
        if fname is not None:  # load the file)
            self.read(fname)

    @staticmethod
    def checkData(data=None):
        if data is None:
            return None
        elif numpy.issubdtype(data.dtype, int):
            return data
        else:
            return data.astype(int)


    def _readheader(self, inStream):
        """
        Read in a header in some CBF format from a string representing binary stuff

        @param inStream: file containing the Cif Binary part.
        @type inStream: opened file.
        """
        self._read_cif_header(inStream)
        self._read_binary_section_header(inStream)

    def _read_cif_header(self, inStream):
        """
        Read in a ASCII CIF header

        @param inStream: file containing the Cif Binary part.
        @type inStream: opened file.
        """
        blocks = []
        last = ""
        header_data = None
        for i in range(16):
            # up to 512*16 = 8k headers
            ablock = inStream.read(PADDING)
            blocks.append(ablock)
            if last:
                extra = len(BINARAY_SECTION)
                extblock = last[-extra:] + ablock
            else:
                extra = 0
                extblock = ablock
            res = extblock.find(BINARAY_SECTION)
            if res >= 0:
                start_cbs = i * PADDING - extra + res
                all_blocks = b"".join(blocks)
                header_data = all_blocks[:start_cbs] + b"CIF Binary Section\n;\n"
                self.cbs = all_blocks[start_cbs:]
                break
            last = ablock
        else:
            header_data = b"".join(blocks) + inStream.read()
        self.cif._parseCIF(header_data)

#        backport contents of the CIF data to the headers
        for key, value in self.cif.items():
            if key == CIF_BINARY_BLOCK_KEY:
                if self.cbs is None:
                    self.cbs = value
            else:
                self.header[key] = (self.cif[key].strip(" \"\n\r\t"))

    def _read_binary_section_header(self, inStream):
        """
        Read the binary section header
        """
        self.start_binary = self.cbs.find(STARTER)
        while self.start_binary < 0:
            self.cbs += inStream.read(PADDING)
            self.start_binary = self.cbs.find(STARTER)
        bin_headers = self.cbs[:self.start_binary]
        lines = bin_headers.split(b"\n")
        for line in lines[1:]:
            if len(line) < 10:
                break
            try:
                key, val = line.split(b':' , 1)
            except ValueError:
                key, val = line.split(b'=' , 1)
            key = key.strip().decode("ASCII")
            self.header[key] = val.strip(b" \"\n\r\t").decode("ASCII")
        missing = []
        for item in MINIMUM_KEYS:
            if item not in self.header:
                missing.append(item)
        if missing:
            logger.info("Mandatory keys missing in CBF file: " + ", ".join(missing))
        # Compute image size
        try:
            self.dim1 = int(self.header['X-Binary-Size-Fastest-Dimension'])
            self.dim2 = int(self.header['X-Binary-Size-Second-Dimension'])
        except:
            raise IOError("CBF file %s is corrupt, no dimensions in it" % fname)
        try:
            self.bytecode = DATA_TYPES[self.header['X-Binary-Element-Type']]
        except KeyError:
            self.bytecode = "int32"
            logger.warning("Defaulting type to int32")
        self.bpp = numpy.dtype(self.bytecode).itemsize

    def read(self, fname, frame=None):
        """
        Read in header into self.header and
            the data   into self.data
        """
        self.filename = fname
        self.header = self.check_header()
        self.resetvals()

        infile = self._open(fname, "rb")
        self._readheader(infile)
        if CIF_BINARY_BLOCK_KEY not in self.cif:
            err = "Not key %s in CIF, no CBF image in %s" % (CIF_BINARY_BLOCK_KEY, fname)
            logger.error(err)
            for kv in self.cif.items():
                print("%s: %s" % kv)
            raise RuntimeError(err)
        if self.cif[CIF_BINARY_BLOCK_KEY] == "CIF Binary Section":
            self.cbs += infile.read(len(STARTER) + int(self.header["X-Binary-Size"]) - len(self.cbs) + self.start_binary)
        else:
            if len(self.cif[CIF_BINARY_BLOCK_KEY]) > int(self.header["X-Binary-Size"]) + self.start_binary + len(STARTER):
                self.cbs = self.cif[CIF_BINARY_BLOCK_KEY][:int(self.header["X-Binary-Size"]) + self.start_binary + len(STARTER)]
            else:
                self.cbs = self.cif[CIF_BINARY_BLOCK_KEY]
        binary_data = self.cbs[self.start_binary + len(STARTER):]
        logger.debug("CBS type %s len %s" % (type(self.cbs), len(self.cbs)))

        if "Content-MD5" in self.header:
                ref = numpy.string_(self.header["Content-MD5"])
                obt = md5sum(binary_data)
                if ref != obt:
                    logger.error("Checksum of binary data mismatch: expected %s, got %s" % (ref, obt))

        if self.header["conversions"] == "x-CBF_BYTE_OFFSET":
            self.data = self._readbinary_byte_offset(binary_data).astype(self.bytecode).reshape((self.dim2, self.dim1))
        else:
            raise Exception(IOError, "Compression scheme not yet supported, please contact the author")

        self.resetvals()
#        # ensure the PIL image is reset
        self.pilimage = None
        return self



    def _readbinary_byte_offset(self, raw_bytes):
        """
        Read in a binary part of an x-CBF_BYTE_OFFSET compressed image

        @param inStream: the binary image (without any CIF decorators)
        @type inStream: python string.
        @return: a linear numpy array without shape and dtype set
        @rtype: numpy array
        """
        myData = decByteOffset(raw_bytes, size=self.dim1 * self.dim2)
        assert len(myData) == self.dim1 * self.dim2
        return myData


    def write(self, fname):
        """
        write the file in CBF format
        @param fname: name of the file
        @type: string
        """
        if self.data is not None:
            self.dim2, self.dim1 = self.data.shape
        else:
            raise RuntimeError("CBF image contains no data")
        binary_blob = compByteOffset(self.data)
#        l = len(binary_blob)
#        if (l % PADDING) != 0:
#            rem = PADDING - (l % PADDING)
#            binary_blob += "\x00" * rem
        dtype = "Unknown"
        for key, value in DATA_TYPES.items():
            if value == self.data.dtype:
                dtype = key
        binary_block = [
                        b"--CIF-BINARY-FORMAT-SECTION--",
                        b"Content-Type: application/octet-stream;",
                        b'     conversions="x-CBF_BYTE_OFFSET"',
                        b'Content-Transfer-Encoding: BINARY',
                        numpy.string_("X-Binary-Size: %d" % (len(binary_blob))),
                        b"X-Binary-ID: 1",
                        numpy.string_('X-Binary-Element-Type: "%s"' % (dtype)),
                        b"X-Binary-Element-Byte-Order: LITTLE_ENDIAN" ,
                        b"Content-MD5: " + md5sum(binary_blob),
                        numpy.string_("X-Binary-Number-of-Elements: %d" % (self.dim1 * self.dim2)),
                        numpy.string_("X-Binary-Size-Fastest-Dimension: %d" % self.dim1),
                        numpy.string_("X-Binary-Size-Second-Dimension: %d" % self.dim2),
                        b"X-Binary-Size-Padding: 1",
                        b"",
                        STARTER + binary_blob,
                        b"",
                        b"--CIF-BINARY-FORMAT-SECTION----"
                        ]

        if "_array_data.header_contents" not in self.header:
            nonCifHeaders = []
        else:
            nonCifHeaders = [i.strip()[2:] for i in self.header["_array_data.header_contents"].split("\n") if i.find("# ") >= 0]

        for key in self.header:
            if key.startswith("_") :
                if key not in self.cif or self.cif[key] != self.header[key]:
                    self.cif[key] = self.header[key]
            elif key.startswith("X-Binary-"):
                pass
            elif key.startswith("Content-"):
                pass
            elif key.startswith("conversions"):
                pass
            elif key.startswith("filename"):
                pass
            elif key in self.header:
                nonCifHeaders.append("%s %s" % (key, self.header[key]))
        if len(nonCifHeaders) > 0:
            self.cif["_array_data.header_contents"] = "\r\n".join(["# %s" % i for i in nonCifHeaders])

        self.cbf = b"\r\n".join(binary_block)
        block = b"\r\n".join([b"", CIF_BINARY_BLOCK_KEY.encode("ASCII"), b";", self.cbf, b";"])
        self.cif.pop(CIF_BINARY_BLOCK_KEY, None)
        with open(fname, "wb") as out_file:
            out_file.write(self.cif.tostring(fname, "\r\n").encode("ASCII"))
            out_file.write(block)

################################################################################
# CIF class
################################################################################
class CIF(dict):
    """
    This is the CIF class, it represents the CIF dictionary;
    and as a a python dictionary thus inherits from the dict built in class.

    keys are always unicode (str in python3)
    values are bytes
    """
    EOL = [numpy.string_(i) for i in ("\r", "\n", "\r\n", "\n\r")]
    BLANK = [numpy.string_(i) for i in (" ", "\t")] + EOL
    SINGLE_QUOTE = numpy.string_("'")
    DOUBLE_QUOTE = numpy.string_('"')
    SEMICOLUMN = numpy.string_(';')
    START_COMMENT = (SINGLE_QUOTE, DOUBLE_QUOTE)
    BINARY_MARKER = numpy.string_("--CIF-BINARY-FORMAT-SECTION--")
    HASH = numpy.string_("#")
    LOOP = numpy.string_("loop_")
    UNDERSCORE = ord("_") if six.PY3 else b"_"
    QUESTIONMARK = ord("?") if six.PY3 else b"?"
    STOP = numpy.string_("stop_")
    GLOBAL = numpy.string_("global_")
    DATA = numpy.string_("data_")
    SAVE = numpy.string_("save_")



    def __init__(self, _strFilename=None):
        """
        Constructor of the class.

        @param _strFilename: the name of the file to open
        @type  _strFilename: filename (str) or file object
        """
        dict.__init__(self)
        self._ordered = []
        if _strFilename is not None:  # load the file)
            self.loadCIF(_strFilename)

    def __setitem__(self, key, value):
        if key not in self._ordered:
            self._ordered.append(key)
        return dict.__setitem__(self, key, value)

    def pop(self, key, default=None):
        if key  in self._ordered:
            self._ordered.remove(key)
        return dict.pop(self, key, default)

    def popitem(self, key, default=None):
        if key  in self._ordered:
            self._ordered.remove(key)
        return dict.popitem(self, key, None)


    def loadCIF(self, _strFilename, _bKeepComment=False):
        """Load the CIF file and populates the CIF dictionary into the object
        @param _strFilename: the name of the file to open
        @type  _strFilename: string
        @param _strFilename: the name of the file to open
        @type  _strFilename: string
        @return: None
        """
        own_fd = False
        if isinstance(_strFilename, (six.binary_type, six.text_type)):
            if os.path.isfile(_strFilename):
                infile = open(_strFilename, "rb")
                own_fd = True
            else:
                raise RuntimeError("CIF.loadCIF: No such file to open: %s" % _strFilename)
        elif "read" in dir(_strFilename):
            infile = _strFilename
        else:
            raise RuntimeError("CIF.loadCIF: what is %s type %s" % (_strFilename, type(_strFilename)))
        if _bKeepComment:
            self._parseCIF(numpy.string_(infile.read()))
        else:
            self._parseCIF(CIF._readCIF(infile))
        if own_fd:
            infile.close()
    readCIF = loadCIF

    @staticmethod
    def isAscii(text):
        """
        Check if all characters in a string are ascii,

        @param _strIn: input string
        @type _strIn: python string
        @return: boolean
        @rtype: boolean
        """
        try:
            text.decode("ascii")
        except UnicodeDecodeError:
            return False
        else:
            return True


    @staticmethod
    def _readCIF(instream):
        """
        - Check if the filename containing the CIF data exists
        - read the cif file
        - removes the comments

        @param instream: opened file object containing the CIF data
        @return: a set of bytes (8-bit string) containing the raw data
        @rtype: string
        """
        if not "read" in dir(instream):
            raise RuntimeError("CIF._readCIF(instream): I expected instream to be an opened file,\
             here I got %s type %s" % (instream, type(instream)))
        out_bytes = numpy.string_("")
        for sLine in instream:
            nline = numpy.string_(sLine)
            pos = nline.find(self.HASH)
            if pos >= 0:
                if self.isAscii(nLine):
                    out_bytes += nLine[:pos] + numpy.string_(os.linesep)
                if pos > 80:
                    logger.warning("This line is too long and could cause problems in PreQuest: %s", sLine)
            else :
                out_bytes += nLine
                if len(sLine.strip()) > 80 :
                    logger.warning("This line is too long and could cause problems in PreQuest: %s", sLine)
        return out_bytes


    def _parseCIF(self, bytes_text):
        """
        - Parses the text of a CIF file
        - Cut it in fields
        - Find all the loops and process
        - Find all the keys and values

        @param bytes_text: the content of the CIF - file
        @type bytes_text: 8-bit string (str in python2 or bytes in python3)
        @return: Nothing, the data are incorporated at the CIF object dictionary
        @rtype: None
        """
        loopidx = []
        looplen = []
        loop = []
        if split_tokens:
            logger.debug("using cython split_tokens")
            fields = split_tokens(bytes_text)
        else:
            logger.debug("using python split")
            fields = self._splitCIF(bytes_text.strip())

        logger.debug("After split got %s fileds" % len(fields))

        for idx, field in enumerate(fields):
            if field.lower() == self.LOOP:
                loopidx.append(idx)
        if loopidx:
            for i in loopidx:
                loopone, length, keys = CIF._analyseOneLoop(fields, i)
                loop.append([keys, loopone])
                looplen.append(length)


            for i in range(len(loopidx) - 1, -1, -1):
                f1 = fields[:loopidx[i]] + fields[loopidx[i] + looplen[i]:]
                fields = f1

            self[self.LOOP.decode("ASCII")] = loop

        for i in range(len(fields) - 1):
            if len(fields[i + 1]) == 0 :
                fields[i + 1] = self.QUESTIONMARK
            if fields[i][0] == self.UNDERSCORE and fields[i + 1][0] != self.UNDERSCORE:
                try:
                    data = fields[i + 1].decode("ASCII")
                except UnicodeError:
                    logger.warning("Unable to decode in ascii: %s" % fields[i + 1])
                    data = fields[i + 1]
                self[(fields[i]).decode("ASCII")] = data

    @classmethod
    def _splitCIF(cls, bytes_text):
        """
        Separate the text in fields as defined in the CIF

        @param bytes_text: the content of the CIF - file
        @type bytes_text:  8-bit string (str in python2 or bytes in python3)
        @return: list of all the fields of the CIF
        @rtype: list
        """
        fields = []
        while True:
            if len(bytes_text) == 0:
                break
            elif bytes_text[0] == cls.SINGLE_QUOTE:
                idx = 0
                finished = False
                while not finished:
                    idx += 1 + bytes_text[idx + 1:].find(cls.SINGLE_QUOTE)
                    if idx >= len(bytes_text) - 1:
                        fields.append(bytes_text[1:-1].strip())
                        bytes_text = numpy.string_("")
                        finished = True
                        break

                    if bytes_text[idx + 1] in cls.BLANK:
                        fields.append(bytes_text[1:idx].strip())
                        tmp_text = bytes_text[idx + 1:]
                        bytes_text = tmp_text.strip()
                        finished = True

            elif bytes_text[0] == cls.DOUBLE_QUOTE:
                idx = 0
                finished = False
                while not finished:
                    idx += 1 + bytes_text[idx + 1:].find(cls.DOUBLE_QUOTE)
                    if idx >= len(bytes_text) - 1:
                        fields.append(bytes_text[1:-1].strip())
                        bytes_text = numpy.string_("")
                        finished = True
                        break

                    if bytes_text[idx + 1] in cls.BLANK:
                        fields.append(bytes_text[1:idx].strip())
                        tmp_text = bytes_text[idx + 1:]
                        bytes_text = tmp_text.strip()
                        finished = True

            elif bytes_text[0] == cls.SEMICOLUMN:
                if bytes_text[1:].strip().find(cls.BINARY_MARKER) == 0:
                    idx = bytes_text[32:].find(cls.BINARY_MARKER)
                    if idx == -1:
                        idx = 0
                    else:
                        idx += 32 + len(cls.BINARY_MARKER)
                else:
                    idx = 0
                finished = False
                while not finished:
                    idx += 1 + bytes_text[idx + 1:].find(cls.SEMICOLUMN)
                    if bytes_text[idx - 1] in cls.EOL:
                        fields.append(bytes_text[1:idx - 1].strip())
                        tmp_text = bytes_text[idx + 1:]
                        bytes_text = tmp_text.strip()
                        finished = True
            else:
                res = bytes_text.split(None, 1)
                if len(res) == 2:
                    first, second = bytes_text.split(None, 1)
                    if cls.isAscii(first):
                        fields.append(first)
                        bytes_text = second.strip()
                        continue
                start_binary = bytes_text.find(cls.BINARY_MARKER)
                if start_binary > 0:
                    end_binary = bytes_text[start_binary + 1:].find(cls.BINARY_MARKER) + start_binary + 1 + len(cls.BINARY_MARKER)
                    fields.append(bytes_text[:end_binary])
                    bytes_text = bytes_text[end_binary:].strip()
                else:
                    fields.append(bytes_text)
                    bytes_text = numpy.string_("")
                    break
        return fields


    @classmethod
    def _analyseOneLoop(cls, fields, start_idx):
        """Processes one loop in the data extraction of the CIF file
        @param fields: list of all the words contained in the cif file
        @type fields: list
        @param start_idx: the starting index corresponding to the "loop_" key
        @type start_idx: integer
        @return: the list of loop dictionaries, the length of the data
            extracted from the fields and the list of all the keys of the loop.
        @rtype: tuple
        """
        loop = []
        keys = []
        i = start_idx + 1
        finished = False
        while not finished:
            if fields[i][0] == cls.UNDERSCORE:
                keys.append(fields[i])
                i += 1
            else:
                finished = True
        data = []
        while True:
            if i >= len(fields):
                break
            elif len(fields[i]) == 0:
                break
            elif fields[i][0] == cls.UNDERSCORE:
                break
            elif fields[i] in (cls.LOOP, cls.STOP, cls.GLOBAL, cls.DATA, cls.SAVE):
                break
            else:
                data.append(fields[i])
                i += 1
        k = 0

        if len(data) < len(keys):
            element = {}
            for j in keys:
                if k < len(data):
                    element[j] = data[k]
                else :
                    element[j] = self.QUESTIONMARK
                k += 1
            loop.append(element)

        else:
            for i in range(len(data) / len(keys)):
                element = {}
                for j in keys:
                    element[j] = data[k]
                    k += 1
                loop.append(element)
        return loop, 1 + len(keys) + len(data), keys






#############################################################################################
########     everything needed to  write a cif file #########################################
#############################################################################################

    def saveCIF(self, _strFilename="test.cif", linesep=os.linesep, binary=False):
        """Transforms the CIF object in string then write it into the given file
        @param _strFilename: the of the file to be written
        @param linesep: line separation used (to force compatibility with windows/unix)
        @param binary: Shall we write the data as binary (True only for imageCIF/CBF)
        @return: None
        """
        if binary:
            mode = "wb"
        else:
            mode = "w"
        try:
            fFile = open(_strFilename, mode)
        except IOError:
            logger.error("Error during the opening of file for write: %s" %
                                                            _strFilename)
            return
        fFile.write(self.tostring(_strFilename, linesep))
        try:
            fFile.close()
        except IOError:
            logger.error("Error during the closing of file for write: %s" % _strFilename)

    def tostring(self, _strFilename=None, linesep=os.linesep):
        """
        Converts a cif dictionnary to a string according to the CIF syntax

        @param _strFilename: the name of the filename to be appended in the header of the CIF file
        @type _strFilename: string
        @param linesep: default line separation: can be "\n" or "\r\n"
        @return: a string that corresponds to the content of the CIF-file.

        """
        lstStrCif = ["# " + i for i in __version__]
        if "_chemical_name_common" in self:
            t = self["_chemical_name_common"].split()[0]
        elif _strFilename is not None:
            t = os.path.splitext(os.path.split(str(_strFilename).strip())[1])[0]
        else:
            t = ""
        lstStrCif.append("data_%s" % (t))
        # first of all get all the keys :
        lKeys = list(self.keys())
        lKeys.sort()
        for key in lKeys[:]:
            if key in self._ordered:
                lKeys.remove(key)
        self._ordered += lKeys

        for sKey in self._ordered:
            if sKey == "loop_":
                continue
            if sKey not in self:
                self._ordered.remove(sKey)
                logger.debug("Skipping key %s from ordered list as no more present in dict")
                continue
            sValue = str(self[sKey])
            if sValue.find("\n") > -1:  # should add value  between ;;
                lLine = [sKey, ";", sValue, ";", ""]
            elif len(sValue.split()) > 1:  # should add value between ''
                sLine = "%s        '%s'" % (sKey, sValue)
                if len(sLine) > 80:
                    lLine = [str(sKey), sValue]
                else:
                    lLine = [sLine]
            else:
                sLine = "%s        %s" % (sKey, sValue)
                if len(sLine) > 80:
                    lLine = [str(sKey), sValue]
                else:
                    lLine = [sLine]
            lstStrCif += lLine
        if "loop_" in self:
            for loop in self["loop_"]:
                lstStrCif.append("loop_ ")
                lKeys = loop[0]
                llData = loop[1]
                lstStrCif += [" %s" % (sKey) for sKey in lKeys]
                for lData in llData:
                    sLine = " "
                    for key in lKeys:
                        sRawValue = lData[key]
                        if sRawValue.find("\n") > -1:  # should add value  between ;;
                            lstStrCif += [sLine, ";", str(sRawValue), ";"]
                            sLine = " "
                        else:
                            if len(sRawValue.split()) > 1:  # should add value between ''
                                value = "'%s'" % (sRawValue)
                            else:
                                value = str(sRawValue)
                            if len(sLine) + len(value) > 78:
                                lstStrCif += [sLine]
                                sLine = " " + value
                            else:
                                sLine += " " + value
                    lstStrCif.append(sLine)
                lstStrCif.append("")
        return linesep.join(lstStrCif)


    def exists(self, sKey):
        """
        Check if the key exists in the CIF and is non empty.
        @param sKey: CIF key
        @type sKey: string
        @param cif: CIF dictionary
        @return: True if the key exists in the CIF dictionary and is non empty
        @rtype: boolean
        """
        bExists = False
        if sKey in self:
            if len(self[sKey]) >= 1:
                if self[sKey][0] not in (self.QUESTIONMARK, numpy.string_(".")):
                    bExists = True
        return bExists

    def existsInLoop(self, sKey):
        """
        Check if the key exists in the CIF dictionary.
        @param sKey: CIF key
        @type sKey: string
        @param cif: CIF dictionary
        @return: True if the key exists in the CIF dictionary and is non empty
        @rtype: boolean
        """
        if not self.exists(self.LOOP):
            return False
        bExists = False
        if not bExists:
            for i in self[self.LOOP]:
                for j in i[0]:
                    if j == sKey:
                        bExists = True
        return bExists

    def loadCHIPLOT(self, _strFilename):
        """
        Load the powder diffraction CHIPLOT file and returns the
        pd_CIF dictionary in the object

        @param _strFilename: the name of the file to open
        @type  _strFilename: string
        @return: the CIF object corresponding to the powder diffraction
        @rtype: dictionary
        """
        if not os.path.isfile(_strFilename):
            errStr = "I cannot find the file %s" % _strFilename
            logger.error(errStr)
            raise IOError(errStr)
        lInFile = open(_strFilename, "r").readlines()
        self["_audit_creation_method"] = 'From 2-D detector using FIT2D and CIFfile'
        self["_pd_meas_scan_method"] = "fixed"
        self["_pd_spec_description"] = lInFile[0].strip()
        try:
            iLenData = int(lInFile[3])
        except ValueError:
            iLenData = None
        lOneLoop = []
        try:
            f2ThetaMin = float(lInFile[4].split()[0])
            last = ""
            for sLine in lInFile[-20:]:
                if sLine.strip() != "":
                    last = sLine.strip()
            f2ThetaMax = float(last.split()[0])
            limitsOK = True

        except (ValueError, IndexError):
            limitsOK = False
            f2ThetaMin = 180.0
            f2ThetaMax = 0
#        print "limitsOK:", limitsOK
        for sLine in lInFile[4:]:
            sCleaned = sLine.split("#")[0].strip()
            data = sCleaned.split()
            if len(data) == 2 :
                if not limitsOK:
                    f2Theta = float(data[0])
                    if f2Theta < f2ThetaMin :
                        f2ThetaMin = f2Theta
                    if f2Theta > f2ThetaMax :
                        f2ThetaMax = f2Theta
                lOneLoop.append({ "_pd_meas_intensity_total": data[1] })
        if not iLenData:
            iLenData = len(lOneLoop)
        assert (iLenData == len(lOneLoop))
        self[ "_pd_meas_2theta_range_inc" ] = "%.4f" % ((f2ThetaMax - f2ThetaMin) / (iLenData - 1))
        if self[ "_pd_meas_2theta_range_inc" ] < 0:
            self[ "_pd_meas_2theta_range_inc" ] = abs (self[ "_pd_meas_2theta_range_inc" ])
            tmp = f2ThetaMax
            f2ThetaMax = f2ThetaMin
            f2ThetaMin = tmp
        self[ "_pd_meas_2theta_range_max" ] = "%.4f" % f2ThetaMax
        self[ "_pd_meas_2theta_range_min" ] = "%.4f" % f2ThetaMin
        self[ "_pd_meas_number_of_points" ] = str(iLenData)
        self[self.LOOP] = [ [ ["_pd_meas_intensity_total" ], lOneLoop ] ]

    @staticmethod
    def LoopHasKey(loop, key):
        "Returns True if the key (string) exist in the array called loop"""
        try:
            loop.index(key)
            return True
        except ValueError:
            return False

cbfimage = CbfImage