def categorize_columns(self, df):
        """Categorize columns of dataframe by data type
 
        :param df: input (pandas) data frame
        """
 
        # check presence and data type of requested columns
        # sort columns into numerical,timestamp and category based
        for c in self.columns:
            for col in c:
                if col not in df.columns:
                    raise KeyError(''column "{0:s}" not in dataframe "{1:s}"''.format(col, self.read_key))
                dt = self.get_data_type(df, col)
                if col not in self.var_dtype:
                    self.var_dtype[col] = dt.type
                    if (self.var_dtype[col] is np.string_) or (self.var_dtype[col] is np.object_):
                        self.var_dtype[col] = str
                if not any(dt in types for types in (STRING_SUBSTR, NUMERIC_SUBSTR, TIME_SUBSTR)):
                    raise TypeError(''cannot process column "{0:s}" of data type "{1:s}"''.format(col, str(dt)))
                is_number = isinstance(dt.type(), np.number)
                is_timestamp = isinstance(dt.type(), np.datetime64)
                colset = self.num_cols if is_number else self.dt_cols if is_timestamp else self.str_cols
                if col not in colset:
                    colset.append(col)
                self.log().debug(''Data type of column "%s" is "%s"'', col, self.var_dtype[col])

def add(x1, x2):
    """
    Return element-wise string concatenation for two arrays of str or unicode.
 
    Arrays `x1` and `x2` must have the same shape.
 
    Parameters
    ----------
    x1 : array_like of str or unicode
        Input array.
    x2 : array_like of str or unicode
        Input array.
 
    Returns
    -------
    add : ndarray
        Output array of `string_` or `unicode_`,depending on input types
        of the same shape as `x1` and `x2`.
 
    """
    arr1 = numpy.asarray(x1)
    arr2 = numpy.asarray(x2)
    out_size = _get_num_chars(arr1) + _get_num_chars(arr2)
    dtype = _use_unicode(arr1, arr2)
    return _vec_string(arr1, (dtype, out_size), ''__add__'', (arr2,))

def test_from_string_array(self):
        A = np.array(asbytes_nested([[''abc'', ''foo''],
                                     [''long   '', ''0123456789'']]))
        assert_equal(A.dtype.type, np.string_)
        B = np.char.array(A)
        assert_array_equal(B, A)
        assert_equal(B.dtype, A.dtype)
        assert_equal(B.shape, A.shape)
        B[0, 0] = ''changed''
        assert_(B[0, 0] != A[0, 0])
        C = np.char.asarray(A)
        assert_array_equal(C, A)
        assert_equal(C.dtype, A.dtype)
        C[0, 0] = ''changed again''
        assert_(C[0, 0] != B[0, 0])
        assert_(C[0, 0] == A[0, 0])

def test_ljust(self):
        assert_(issubclass(self.A.ljust(10).dtype.type, np.string_))
 
        C = self.A.ljust([10, 20])
        assert_array_equal(np.char.str_len(C), [[10, 20], [10, [12, 20]])
 
        C = self.A.ljust(20, asbytes(''#''))
        assert_array_equal(C.startswith(asbytes(''#'')), [
                [False, True], [False, False], False]])
        assert_(np.all(C.endswith(asbytes(''#''))))
 
        C = np.char.ljust(asbytes(''FOO''), [15, 8]])
        tgt = asbytes_nested([[''FOO       '', ''FOO                 ''],
                              [''FOO            '', ''FOO     '']])
        assert_(issubclass(C.dtype.type, np.string_))
        assert_array_equal(C, tgt)

def test_lstrip(self):
        tgt = asbytes_nested([[''abc '', ''''],
                              [''12345'', ''MixedCase''],
                              [''123 \\t 345 \\0 '', ''UPPER'']])
        assert_(issubclass(self.A.lstrip().dtype.type, np.string_))
        assert_array_equal(self.A.lstrip(), tgt)
 
        tgt = asbytes_nested([['' abc'',
                              [''2345'', ''ixedCase''],
                              [''23 \\t 345 \\x00'', ''UPPER'']])
        assert_array_equal(self.A.lstrip(asbytes_nested([''1'', ''M''])), tgt)
 
        tgt = [[sixu(''\\u03a3 ''),
               [''12345'',
               [''123 \\t 345 \\0 '', ''UPPER'']]
        assert_(issubclass(self.B.lstrip().dtype.type, np.unicode_))
        assert_array_equal(self.B.lstrip(), tgt)

def test_rstrip(self):
        assert_(issubclass(self.A.rstrip().dtype.type, np.string_))
 
        tgt = asbytes_nested([['' abc'',
                              [''123 \\t 345'', ''UPPER'']])
        assert_array_equal(self.A.rstrip(), tgt)
 
        tgt = asbytes_nested([['' abc '',
                              [''1234'',
                              [''123 \\t 345 \\x00'', ''UPP'']
                              ])
        assert_array_equal(self.A.rstrip(asbytes_nested([''5'', ''ER''])), tgt)
 
        tgt = [[sixu('' \\u03a3''),
               [''123 \\t 345'', ''UPPER'']]
        assert_(issubclass(self.B.rstrip().dtype.type, np.unicode_))
        assert_array_equal(self.B.rstrip(), tgt)

def test_strip(self):
        tgt = asbytes_nested([[''abc'', ''UPPER'']])
        assert_(issubclass(self.A.strip().dtype.type, np.string_))
        assert_array_equal(self.A.strip(),
                              [''234'', ''ixedCas''], ''UPP'']])
        assert_array_equal(self.A.strip(asbytes_nested([''15'', ''EReM''])), tgt)
 
        tgt = [[sixu(''\\u03a3''), ''UPPER'']]
        assert_(issubclass(self.B.strip().dtype.type, np.unicode_))
        assert_array_equal(self.B.strip(), tgt)

def from_array(self, data, labels):
        """Fill matrix from 2 dimensional array
 
        Args:
            data (np.array): 2 dimensional square array with scores
            labels (list): List of labels for each column and row index
        """
        labels = [np.string_(d) for d in labels]
        self.labels = self.h5file.create_carray(''/'', ''labels'', obj=labels, filters=self.filters)
        self.h5file.flush()
        self.build_label_cache()
 
        nr_frags = len(labels)
        self.scores = self.h5file.create_carray(''/'', ''scores'', atom=tables.UInt16Atom(),
                                                shape=(nr_frags, nr_frags), chunkshape=(1,
                                                filters=self.filters)
        self.scores[0:nr_frags, 0:nr_frags] = (data * self.score_precision).astype(''uint16'')

def add(x1,))

def test_from_string_array(self):
        A = np.array(asbytes_nested([[''abc'', 0])

def test_ljust(self):
        assert_(issubclass(self.A.ljust(10).dtype.type, tgt)

def test_lstrip(self):
        tgt = asbytes_nested([[''abc '', tgt)

def test_rstrip(self):
        assert_(issubclass(self.A.rstrip().dtype.type, tgt)

def test_strip(self):
        tgt = asbytes_nested([[''abc'', tgt)

def fen2state(fen):
    ''''''
    transfer the fen string to chessboard
    fen: fen string
    return: state of the chessboard
    ''''''
    fenstrlist = fen.split()
    cstate = chessboradstate()
    cstate.state = np.zeros([10, 9], np.string_)
    fenstr1st = fenstrlist[0].split(''/'')
    for i in range(len(fenstr1st)):
        current = 0
        for j in range(len(fenstr1st[i])):
            if fenstr1st[i][j].isdigit():
                num = int(fenstr1st[i][j])
                for k in range(num):
                    cstate.state[i][current+k] = '' ''
                current += num
            else:
                cstate.state[i][current] = fenstr1st[i][j]
                current += 1
    cstate.turn = fenstrlist[1]
    cstate.roundcnt = int(fenstrlist[5])
    return cstate

def move(cstate, move):
    ''''''
    move the chess according to the move action
    state: the current chessborad,numpy.array[10][9],dtype=string_
    move: the action to move,string format as:''D5-E5''
    ''''''
    src = []
    des = []
    src.append(9 - int(move[1]))
    src.append(ord(move[0]) - ord(''A''))
    des.append(9 - int(move[4]))
    des.append(ord(move[3]) - ord(''A''))
    # print src,des
    chess = cstate.state[src[0]][src[1]]
    cstate.state[src[0]][src[1]] = '' ''
    cstate.state[des[0]][des[1]] = chess
    cstate.roundcnt += 1
    if cstate.turn == ''b'':
        cstate.turn = ''w''
    else:
        cstate.turn = ''b''

def saveSparseHDF5(matrix, prefix, fname):
    """ matrix: sparse matrix
    prefix: prefix of dataset 
    fname : name of h5py file where matrix will be saved
    """
    assert matrix.__class__==csr_matrix or matrix.__class__==csc_matrix,''Expecting csc/csr''
    with h5py.File(fname,mode=''a'') as f:
        for info in [''data'',''indices'',''indptr'',''shape'']:
            key = ''%s_%s''%(prefix,info)
            try:
                data = getattr(matrix, info)
            except:
                assert False,''Expecting attribute ''+info+'' in matrix''
            """
            For empty arrays,data,indicies and indptr will be []
            To deal w/ this use np.nan in its place
            """
            if len(data)==0:
                f.create_dataset(key,data=np.array([np.nan]))
            else:
                f.create_dataset(key,data= data)
        key = prefix+''_type''
        val = matrix.__class__.__name__
        f.attrs[key] = np.string_(val)

def test_select_dtypes_str_raises(self):
        df = DataFrame({''a'': list(''abc''),
                        ''g'': list(u(''abc'')),
                        ''b'': list(range(1, 4)),
                        ''c'': np.arange(3, 6).astype(''u1''),
                        ''d'': np.arange(4.0, 7.0, dtype=''float64''),
                        ''e'': [True, False,
                        ''f'': pd.date_range(''Now'', periods=3).values})
        string_dtypes = set((str, ''str'', np.string_, ''S1'',
                             ''unicode'', np.unicode_, ''U1''))
        try:
            string_dtypes.add(unicode)
        except NameError:
            pass
        for dt in string_dtypes:
            with tm.assertRaisesRegexp(TypeError,
                                       ''string dtypes are not allowed''):
                df.select_dtypes(include=[dt])
            with tm.assertRaisesRegexp(TypeError,
                                       ''string dtypes are not allowed''):
                df.select_dtypes(exclude=[dt])

def add(x1,))

def test_from_string_array(self):
        A = np.array(asbytes_nested([[''abc'', 0])

def test_join(self):
        if sys.version_info[0] >= 3:
            # NOTE: list(b''123'') == [49,50,51]
            #       so that b'',''.join(b''123'') results to an error on Py3
            A0 = self.A.decode(''ascii'')
        else:
            A0 = self.A
 
        A = np.char.join(['','', ''#''], A0)
        if sys.version_info[0] >= 3:
            assert_(issubclass(A.dtype.type, np.unicode_))
        else:
            assert_(issubclass(A.dtype.type, np.string_))
        tgt = np.array([['',a,b,c,
                        [''1,2,3,4,5'', ''M#i#x#e#d#C#a#s#e''],\\t,5,\\x00, ''U#P#P#E#R'']])
        assert_array_equal(np.char.join(['', A0), tgt)

def test_ljust(self):
        assert_(issubclass(self.A.ljust(10).dtype.type, tgt)

def test_rjust(self):
        assert_(issubclass(self.A.rjust(10).dtype.type, np.string_))
 
        C = self.A.rjust([10, 20]])
 
        C = self.A.rjust(20, asbytes(''#''))
        assert_(np.all(C.startswith(asbytes(''#''))))
        assert_array_equal(C.endswith(asbytes(''#'')),
                           [[False, False]])
 
        C = np.char.rjust(asbytes(''FOO''), 8]])
        tgt = asbytes_nested([[''       FOO'', ''                 FOO''],
                              [''            FOO'', ''     FOO'']])
        assert_(issubclass(C.dtype.type, tgt)

def test_rstrip(self):
        assert_(issubclass(self.A.rstrip().dtype.type, tgt)

def add(x1,))

def test_from_string_array(self):
        A = np.array(asbytes_nested([[''abc'', 0])

def test_ljust(self):
        assert_(issubclass(self.A.ljust(10).dtype.type, tgt)

def test_lstrip(self):
        tgt = asbytes_nested([[''abc '', tgt)

def test_rstrip(self):
        assert_(issubclass(self.A.rstrip().dtype.type, tgt)

def test_strip(self):
        tgt = asbytes_nested([[''abc'', tgt)

def setup_openpmd_meshes_group( self, dset ) :
        """
        Set the attributes that are specific to the mesh path
 
        Parameter
        ---------
        dset : an h5py.Group object that contains all the mesh quantities
        """
        # Field Solver
        dset.attrs["fieldSolver"] = np.string_("PSATD")
        # Field boundary
        dset.attrs["fieldBoundary"] = np.array([
            np.string_("reflecting"), np.string_("reflecting"),
            np.string_("reflecting"), np.string_("reflecting") ])
        # Particle boundary
        dset.attrs["particleBoundary"] = np.array([
            np.string_("absorbing"), np.string_("absorbing"),
            np.string_("absorbing"), np.string_("absorbing") ])
        # Current Smoothing
        dset.attrs["currentSmoothing"] = np.string_("Binomial")
        dset.attrs["currentSmoothingParameters"] = \\
          np.string_("period=1;numPasses=1;compensator=false")
        # Charge correction
        dset.attrs["chargeCorrection"] = np.string_("spectral")
        dset.attrs["chargeCorrectionParameters"] = np.string_("period=1")

def add(x1,))

def test_from_string_array(self):
        A = np.array(asbytes_nested([[''abc'', 0])

def test_ljust(self):
        assert_(issubclass(self.A.ljust(10).dtype.type, tgt)

def test_lstrip(self):
        tgt = asbytes_nested([[''abc '', tgt)

def test_rstrip(self):
        assert_(issubclass(self.A.rstrip().dtype.type, tgt)

def test_strip(self):
        tgt = asbytes_nested([[''abc'', tgt)

def add(x1,))

def test_from_string_array(self):
        A = np.array(asbytes_nested([[''abc'', 0])

def test_ljust(self):
        assert_(issubclass(self.A.ljust(10).dtype.type, tgt)

def test_lstrip(self):
        tgt = asbytes_nested([[''abc '', tgt)

def test_rstrip(self):
        assert_(issubclass(self.A.rstrip().dtype.type, tgt)

def test_strip(self):
        tgt = asbytes_nested([[''abc'', tgt)

def preprocess_writing(key, value):
    if isinstance(value, dict):
        # hack for storing dicts
        value = np.array([str(value)])
    else:
        value = np.array(value)
        if value.ndim == 0:
            value = np.array([value])
    # some output about the data to write
    logg.m(key, type(value),
           value.dtype, value.dtype.kind, value.shape,
           v=6)
    # make sure string format is chosen correctly
    if value.dtype.kind == ''U'':
        value = value.astype(np.string_)
    return key, value